From d60f350cf8b7dc361f9c0b3ab87daac86c79cd98 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Tue, 27 May 2025 23:11:44 -0700 Subject: [PATCH 01/41] feat: add support for -c/--config to override individual config items (#1137) This PR introduces support for `-c`/`--config` so users can override individual config values on the command line using `--config name=value`. Example: ``` codex --config model=o4-mini ``` Making it possible to set arbitrary config values on the command line results in a more flexible configuration scheme and makes it easier to provide single-line examples that can be copy-pasted from documentation. Effectively, it means there are four levels of configuration for some values: - Default value (e.g., `model` currently defaults to `o4-mini`) - Value in `config.toml` (e.g., user could override the default to be `model = "o3"` in their `config.toml`) - Specifying `-c` or `--config` to override `model` (e.g., user can include `-c model=o3` in their list of args to Codex) - If available, a config-specific flag can be used, which takes precedence over `-c` (e.g., user can specify `--model o3` in their list of args to Codex) Now that it is possible to specify anything that could be configured in `config.toml` on the command line using `-c`, we do not need to have a custom flag for every possible config option (which can clutter the output of `--help`). To that end, as part of this PR, we drop support for the `--disable-response-storage` flag, as users can now specify `-c disable_response_storage=true` to get the equivalent functionality. Under the hood, this works by loading the `config.toml` into a `toml::Value`. Then for each `key=value`, we create a small synthetic TOML file with `value` so that we can run the TOML parser to get the equivalent `toml::Value`. We then parse `key` to determine the point in the original `toml::Value` to do the insert/replace. Once all of the overrides from `-c` args have been applied, the `toml::Value` is deserialized into a `ConfigToml` and then the `ConfigOverrides` are applied, as before. --- codex-rs/Cargo.lock | 3 + codex-rs/cli/src/debug_sandbox.rs | 21 ++- codex-rs/cli/src/lib.rs | 7 + codex-rs/cli/src/main.rs | 35 +++- codex-rs/cli/src/proto.rs | 15 +- codex-rs/common/Cargo.toml | 4 +- codex-rs/common/src/config_override.rs | 170 +++++++++++++++++++ codex-rs/common/src/lib.rs | 6 + codex-rs/core/src/config.rs | 147 +++++++++++----- codex-rs/core/src/config_types.rs | 2 +- codex-rs/exec/src/cli.rs | 6 +- codex-rs/exec/src/lib.rs | 18 +- codex-rs/exec/src/main.rs | 21 ++- codex-rs/mcp-server/Cargo.toml | 1 + codex-rs/mcp-server/src/codex_tool_config.rs | 37 ++-- codex-rs/mcp-server/src/json_to_toml.rs | 84 +++++++++ codex-rs/mcp-server/src/lib.rs | 1 + codex-rs/tui/src/cli.rs | 6 +- codex-rs/tui/src/lib.rs | 17 +- codex-rs/tui/src/main.rs | 19 ++- 20 files changed, 522 insertions(+), 98 deletions(-) create mode 100644 codex-rs/common/src/config_override.rs create mode 100644 codex-rs/mcp-server/src/json_to_toml.rs diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 309c671e74..8f1762cac6 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -506,6 +506,8 @@ version = "0.0.0" dependencies = [ "clap", "codex-core", + "serde", + "toml", ] [[package]] @@ -634,6 +636,7 @@ dependencies = [ "serde", "serde_json", "tokio", + "toml", "tracing", "tracing-subscriber", ] diff --git a/codex-rs/cli/src/debug_sandbox.rs b/codex-rs/cli/src/debug_sandbox.rs index c09cee020a..deacca5f28 100644 --- a/codex-rs/cli/src/debug_sandbox.rs +++ b/codex-rs/cli/src/debug_sandbox.rs @@ -1,5 +1,6 @@ use std::path::PathBuf; +use codex_common::CliConfigOverrides; use codex_common::SandboxPermissionOption; use codex_core::config::Config; use codex_core::config::ConfigOverrides; @@ -20,12 +21,14 @@ pub async fn run_command_under_seatbelt( let SeatbeltCommand { full_auto, sandbox, + config_overrides, command, } = command; run_command_under_sandbox( full_auto, sandbox, command, + config_overrides, codex_linux_sandbox_exe, SandboxType::Seatbelt, ) @@ -39,12 +42,14 @@ pub async fn run_command_under_landlock( let LandlockCommand { full_auto, sandbox, + config_overrides, command, } = command; run_command_under_sandbox( full_auto, sandbox, command, + config_overrides, codex_linux_sandbox_exe, SandboxType::Landlock, ) @@ -60,16 +65,22 @@ async fn run_command_under_sandbox( full_auto: bool, sandbox: SandboxPermissionOption, command: Vec, + config_overrides: CliConfigOverrides, codex_linux_sandbox_exe: Option, sandbox_type: SandboxType, ) -> anyhow::Result<()> { let sandbox_policy = create_sandbox_policy(full_auto, sandbox); let cwd = std::env::current_dir()?; - let config = Config::load_with_overrides(ConfigOverrides { - sandbox_policy: Some(sandbox_policy), - codex_linux_sandbox_exe, - ..Default::default() - })?; + let config = Config::load_with_cli_overrides( + config_overrides + .parse_overrides() + .map_err(anyhow::Error::msg)?, + ConfigOverrides { + sandbox_policy: Some(sandbox_policy), + codex_linux_sandbox_exe, + ..Default::default() + }, + )?; let stdio_policy = StdioPolicy::Inherit; let env = create_env(&config.shell_environment_policy); diff --git a/codex-rs/cli/src/lib.rs b/codex-rs/cli/src/lib.rs index bf85c98c8e..0730a919d7 100644 --- a/codex-rs/cli/src/lib.rs +++ b/codex-rs/cli/src/lib.rs @@ -3,6 +3,7 @@ mod exit_status; pub mod proto; use clap::Parser; +use codex_common::CliConfigOverrides; use codex_common::SandboxPermissionOption; #[derive(Debug, Parser)] @@ -14,6 +15,9 @@ pub struct SeatbeltCommand { #[clap(flatten)] pub sandbox: SandboxPermissionOption, + #[clap(skip)] + pub config_overrides: CliConfigOverrides, + /// Full command args to run under seatbelt. #[arg(trailing_var_arg = true)] pub command: Vec, @@ -28,6 +32,9 @@ pub struct LandlockCommand { #[clap(flatten)] pub sandbox: SandboxPermissionOption, + #[clap(skip)] + pub config_overrides: CliConfigOverrides, + /// Full command args to run under landlock. #[arg(trailing_var_arg = true)] pub command: Vec, diff --git a/codex-rs/cli/src/main.rs b/codex-rs/cli/src/main.rs index 8f44962e6d..1c362d2a48 100644 --- a/codex-rs/cli/src/main.rs +++ b/codex-rs/cli/src/main.rs @@ -2,6 +2,7 @@ use clap::Parser; use codex_cli::LandlockCommand; use codex_cli::SeatbeltCommand; use codex_cli::proto; +use codex_common::CliConfigOverrides; use codex_exec::Cli as ExecCli; use codex_tui::Cli as TuiCli; use std::path::PathBuf; @@ -19,6 +20,9 @@ use crate::proto::ProtoCli; subcommand_negates_reqs = true )] struct MultitoolCli { + #[clap(flatten)] + pub config_overrides: CliConfigOverrides, + #[clap(flatten)] interactive: TuiCli, @@ -73,28 +77,34 @@ async fn cli_main(codex_linux_sandbox_exe: Option) -> anyhow::Result<() match cli.subcommand { None => { - codex_tui::run_main(cli.interactive, codex_linux_sandbox_exe)?; + let mut tui_cli = cli.interactive; + prepend_config_flags(&mut tui_cli.config_overrides, cli.config_overrides); + codex_tui::run_main(tui_cli, codex_linux_sandbox_exe)?; } - Some(Subcommand::Exec(exec_cli)) => { + Some(Subcommand::Exec(mut exec_cli)) => { + prepend_config_flags(&mut exec_cli.config_overrides, cli.config_overrides); codex_exec::run_main(exec_cli, codex_linux_sandbox_exe).await?; } Some(Subcommand::Mcp) => { codex_mcp_server::run_main(codex_linux_sandbox_exe).await?; } - Some(Subcommand::Proto(proto_cli)) => { + Some(Subcommand::Proto(mut proto_cli)) => { + prepend_config_flags(&mut proto_cli.config_overrides, cli.config_overrides); proto::run_main(proto_cli).await?; } Some(Subcommand::Debug(debug_args)) => match debug_args.cmd { - DebugCommand::Seatbelt(seatbelt_command) => { + DebugCommand::Seatbelt(mut seatbelt_cli) => { + prepend_config_flags(&mut seatbelt_cli.config_overrides, cli.config_overrides); codex_cli::debug_sandbox::run_command_under_seatbelt( - seatbelt_command, + seatbelt_cli, codex_linux_sandbox_exe, ) .await?; } - DebugCommand::Landlock(landlock_command) => { + DebugCommand::Landlock(mut landlock_cli) => { + prepend_config_flags(&mut landlock_cli.config_overrides, cli.config_overrides); codex_cli::debug_sandbox::run_command_under_landlock( - landlock_command, + landlock_cli, codex_linux_sandbox_exe, ) .await?; @@ -104,3 +114,14 @@ async fn cli_main(codex_linux_sandbox_exe: Option) -> anyhow::Result<() Ok(()) } + +/// Prepend root-level overrides so they have lower precedence than +/// CLI-specific ones specified after the subcommand (if any). +fn prepend_config_flags( + subcommand_config_overrides: &mut CliConfigOverrides, + cli_config_overrides: CliConfigOverrides, +) { + subcommand_config_overrides + .raw_overrides + .splice(0..0, cli_config_overrides.raw_overrides); +} diff --git a/codex-rs/cli/src/proto.rs b/codex-rs/cli/src/proto.rs index 6dbe049cc3..148699552a 100644 --- a/codex-rs/cli/src/proto.rs +++ b/codex-rs/cli/src/proto.rs @@ -2,6 +2,7 @@ use std::io::IsTerminal; use std::sync::Arc; use clap::Parser; +use codex_common::CliConfigOverrides; use codex_core::Codex; use codex_core::config::Config; use codex_core::config::ConfigOverrides; @@ -13,9 +14,12 @@ use tracing::error; use tracing::info; #[derive(Debug, Parser)] -pub struct ProtoCli {} +pub struct ProtoCli { + #[clap(skip)] + pub config_overrides: CliConfigOverrides, +} -pub async fn run_main(_opts: ProtoCli) -> anyhow::Result<()> { +pub async fn run_main(opts: ProtoCli) -> anyhow::Result<()> { if std::io::stdin().is_terminal() { anyhow::bail!("Protocol mode expects stdin to be a pipe, not a terminal"); } @@ -24,7 +28,12 @@ pub async fn run_main(_opts: ProtoCli) -> anyhow::Result<()> { .with_writer(std::io::stderr) .init(); - let config = Config::load_with_overrides(ConfigOverrides::default())?; + let ProtoCli { config_overrides } = opts; + let overrides_vec = config_overrides + .parse_overrides() + .map_err(anyhow::Error::msg)?; + + let config = Config::load_with_cli_overrides(overrides_vec, ConfigOverrides::default())?; let ctrl_c = notify_on_sigint(); let (codex, _init_id) = Codex::spawn(config, ctrl_c.clone()).await?; let codex = Arc::new(codex); diff --git a/codex-rs/common/Cargo.toml b/codex-rs/common/Cargo.toml index 95e4a53182..b4b658dabf 100644 --- a/codex-rs/common/Cargo.toml +++ b/codex-rs/common/Cargo.toml @@ -9,8 +9,10 @@ workspace = true [dependencies] clap = { version = "4", features = ["derive", "wrap_help"], optional = true } codex-core = { path = "../core" } +toml = { version = "0.8", optional = true } +serde = { version = "1", optional = true } [features] # Separate feature so that `clap` is not a mandatory dependency. -cli = ["clap"] +cli = ["clap", "toml", "serde"] elapsed = [] diff --git a/codex-rs/common/src/config_override.rs b/codex-rs/common/src/config_override.rs new file mode 100644 index 0000000000..bd2c036940 --- /dev/null +++ b/codex-rs/common/src/config_override.rs @@ -0,0 +1,170 @@ +//! Support for `-c key=value` overrides shared across Codex CLI tools. +//! +//! This module provides a [`CliConfigOverrides`] struct that can be embedded +//! into a `clap`-derived CLI struct using `#[clap(flatten)]`. Each occurrence +//! of `-c key=value` (or `--config key=value`) will be collected as a raw +//! string. Helper methods are provided to convert the raw strings into +//! key/value pairs as well as to apply them onto a mutable +//! `serde_json::Value` representing the configuration tree. + +use clap::ArgAction; +use clap::Parser; +use serde::de::Error as SerdeError; +use toml::Value; + +/// CLI option that captures arbitrary configuration overrides specified as +/// `-c key=value`. It intentionally keeps both halves **unparsed** so that the +/// calling code can decide how to interpret the right-hand side. +#[derive(Parser, Debug, Default, Clone)] +pub struct CliConfigOverrides { + /// Override a configuration value that would otherwise be loaded from + /// `~/.codex/config.toml`. Use a dotted path (`foo.bar.baz`) to override + /// nested values. The `value` portion is parsed as JSON. If it fails to + /// parse as JSON, the raw string is used as a literal. + /// + /// Examples: + /// - `-c model="o4-mini"` + /// - `-c 'sandbox_permissions=["disk-full-read-access"]'` + /// - `-c shell_environment_policy.inherit=all` + #[arg( + short = 'c', + long = "config", + value_name = "key=value", + action = ArgAction::Append, + global = true, + )] + pub raw_overrides: Vec, +} + +impl CliConfigOverrides { + /// Parse the raw strings captured from the CLI into a list of `(path, + /// value)` tuples where `value` is a `serde_json::Value`. + pub fn parse_overrides(&self) -> Result, String> { + self.raw_overrides + .iter() + .map(|s| { + // Only split on the *first* '=' so values are free to contain + // the character. + let mut parts = s.splitn(2, '='); + let key = match parts.next() { + Some(k) => k.trim(), + None => return Err("Override missing key".to_string()), + }; + let value_str = parts + .next() + .ok_or_else(|| format!("Invalid override (missing '='): {s}"))? + .trim(); + + if key.is_empty() { + return Err(format!("Empty key in override: {s}")); + } + + // Attempt to parse as JSON. If that fails, treat it as a raw + // string. This allows convenient usage such as + // `-c model=o4-mini` without the quotes. + let value: Value = match parse_toml_value(value_str) { + Ok(v) => v, + Err(_) => Value::String(value_str.to_string()), + }; + + Ok((key.to_string(), value)) + }) + .collect() + } + + /// Apply all parsed overrides onto `target`. Intermediate objects will be + /// created as necessary. Values located at the destination path will be + /// replaced. + pub fn apply_on_value(&self, target: &mut Value) -> Result<(), String> { + let overrides = self.parse_overrides()?; + for (path, value) in overrides { + apply_single_override(target, &path, value); + } + Ok(()) + } +} + +/// Apply a single override onto `root`, creating intermediate objects as +/// necessary. +fn apply_single_override(root: &mut Value, path: &str, value: Value) { + use toml::value::Table; + + let parts: Vec<&str> = path.split('.').collect(); + let mut current = root; + + for (i, part) in parts.iter().enumerate() { + let is_last = i == parts.len() - 1; + + if is_last { + match current { + Value::Table(tbl) => { + tbl.insert((*part).to_string(), value); + } + _ => { + let mut tbl = Table::new(); + tbl.insert((*part).to_string(), value); + *current = Value::Table(tbl); + } + } + return; + } + + // Traverse or create intermediate table. + match current { + Value::Table(tbl) => { + current = tbl + .entry((*part).to_string()) + .or_insert_with(|| Value::Table(Table::new())); + } + _ => { + *current = Value::Table(Table::new()); + if let Value::Table(tbl) = current { + current = tbl + .entry((*part).to_string()) + .or_insert_with(|| Value::Table(Table::new())); + } + } + } + } +} + +fn parse_toml_value(raw: &str) -> Result { + let wrapped = format!("_x_ = {raw}"); + let table: toml::Table = toml::from_str(&wrapped)?; + table + .get("_x_") + .cloned() + .ok_or_else(|| SerdeError::custom("missing sentinel key")) +} + +#[cfg(all(test, feature = "cli"))] +#[allow(clippy::expect_used, clippy::unwrap_used)] +mod tests { + use super::*; + + #[test] + fn parses_basic_scalar() { + let v = parse_toml_value("42").expect("parse"); + assert_eq!(v.as_integer(), Some(42)); + } + + #[test] + fn fails_on_unquoted_string() { + assert!(parse_toml_value("hello").is_err()); + } + + #[test] + fn parses_array() { + let v = parse_toml_value("[1, 2, 3]").expect("parse"); + let arr = v.as_array().expect("array"); + assert_eq!(arr.len(), 3); + } + + #[test] + fn parses_inline_table() { + let v = parse_toml_value("{a = 1, b = 2}").expect("parse"); + let tbl = v.as_table().expect("table"); + assert_eq!(tbl.get("a").unwrap().as_integer(), Some(1)); + assert_eq!(tbl.get("b").unwrap().as_integer(), Some(2)); + } +} diff --git a/codex-rs/common/src/lib.rs b/codex-rs/common/src/lib.rs index 2533718883..c2283640cb 100644 --- a/codex-rs/common/src/lib.rs +++ b/codex-rs/common/src/lib.rs @@ -8,3 +8,9 @@ pub mod elapsed; pub use approval_mode_cli_arg::ApprovalModeCliArg; #[cfg(feature = "cli")] pub use approval_mode_cli_arg::SandboxPermissionOption; + +#[cfg(any(feature = "cli", test))] +mod config_override; + +#[cfg(feature = "cli")] +pub use config_override::CliConfigOverrides; diff --git a/codex-rs/core/src/config.rs b/codex-rs/core/src/config.rs index d643d00660..b6871da153 100644 --- a/codex-rs/core/src/config.rs +++ b/codex-rs/core/src/config.rs @@ -16,6 +16,7 @@ use serde::Deserialize; use std::collections::HashMap; use std::path::Path; use std::path::PathBuf; +use toml::Value as TomlValue; /// Maximum number of bytes of the documentation that will be embedded. Larger /// files are *silently truncated* to this size so we do not take up too much of @@ -108,6 +109,108 @@ pub struct Config { pub codex_linux_sandbox_exe: Option, } +impl Config { + /// Load configuration with *generic* CLI overrides (`-c key=value`) applied + /// **in between** the values parsed from `config.toml` and the + /// strongly-typed overrides specified via [`ConfigOverrides`]. + /// + /// The precedence order is therefore: `config.toml` < `-c` overrides < + /// `ConfigOverrides`. + pub fn load_with_cli_overrides( + cli_overrides: Vec<(String, TomlValue)>, + overrides: ConfigOverrides, + ) -> std::io::Result { + // Resolve the directory that stores Codex state (e.g. ~/.codex or the + // value of $CODEX_HOME) so we can embed it into the resulting + // `Config` instance. + let codex_home = find_codex_home()?; + + // Step 1: parse `config.toml` into a generic JSON value. + let mut root_value = load_config_as_toml(&codex_home)?; + + // Step 2: apply the `-c` overrides. + for (path, value) in cli_overrides.into_iter() { + apply_toml_override(&mut root_value, &path, value); + } + + // Step 3: deserialize into `ConfigToml` so that Serde can enforce the + // correct types. + let cfg: ConfigToml = root_value.try_into().map_err(|e| { + tracing::error!("Failed to deserialize overridden config: {e}"); + std::io::Error::new(std::io::ErrorKind::InvalidData, e) + })?; + + // Step 4: merge with the strongly-typed overrides. + Self::load_from_base_config_with_overrides(cfg, overrides, codex_home) + } +} + +/// Read `CODEX_HOME/config.toml` and return it as a generic TOML value. Returns +/// an empty TOML table when the file does not exist. +fn load_config_as_toml(codex_home: &Path) -> std::io::Result { + let config_path = codex_home.join("config.toml"); + match std::fs::read_to_string(&config_path) { + Ok(contents) => match toml::from_str::(&contents) { + Ok(val) => Ok(val), + Err(e) => { + tracing::error!("Failed to parse config.toml: {e}"); + Err(std::io::Error::new(std::io::ErrorKind::InvalidData, e)) + } + }, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => { + tracing::info!("config.toml not found, using defaults"); + Ok(TomlValue::Table(Default::default())) + } + Err(e) => { + tracing::error!("Failed to read config.toml: {e}"); + Err(e) + } + } +} + +/// Apply a single dotted-path override onto a TOML value. +fn apply_toml_override(root: &mut TomlValue, path: &str, value: TomlValue) { + use toml::value::Table; + + let segments: Vec<&str> = path.split('.').collect(); + let mut current = root; + + for (idx, segment) in segments.iter().enumerate() { + let is_last = idx == segments.len() - 1; + + if is_last { + match current { + TomlValue::Table(table) => { + table.insert(segment.to_string(), value); + } + _ => { + let mut table = Table::new(); + table.insert(segment.to_string(), value); + *current = TomlValue::Table(table); + } + } + return; + } + + // Traverse or create intermediate object. + match current { + TomlValue::Table(table) => { + current = table + .entry(segment.to_string()) + .or_insert_with(|| TomlValue::Table(Table::new())); + } + _ => { + *current = TomlValue::Table(Table::new()); + if let TomlValue::Table(tbl) = current { + current = tbl + .entry(segment.to_string()) + .or_insert_with(|| TomlValue::Table(Table::new())); + } + } + } + } +} + /// Base config deserialized from ~/.codex/config.toml. #[derive(Deserialize, Debug, Clone, Default)] pub struct ConfigToml { @@ -171,29 +274,6 @@ pub struct ConfigToml { pub tui: Option, } -impl ConfigToml { - /// Attempt to parse the file at `~/.codex/config.toml`. If it does not - /// exist, return a default config. Though if it exists and cannot be - /// parsed, report that to the user and force them to fix it. - fn load_from_toml(codex_home: &Path) -> std::io::Result { - let config_toml_path = codex_home.join("config.toml"); - match std::fs::read_to_string(&config_toml_path) { - Ok(contents) => toml::from_str::(&contents).map_err(|e| { - tracing::error!("Failed to parse config.toml: {e}"); - std::io::Error::new(std::io::ErrorKind::InvalidData, e) - }), - Err(e) if e.kind() == std::io::ErrorKind::NotFound => { - tracing::info!("config.toml not found, using defaults"); - Ok(Self::default()) - } - Err(e) => { - tracing::error!("Failed to read config.toml: {e}"); - Err(e) - } - } - } -} - fn deserialize_sandbox_permissions<'de, D>( deserializer: D, ) -> Result>, D::Error> @@ -227,28 +307,12 @@ pub struct ConfigOverrides { pub cwd: Option, pub approval_policy: Option, pub sandbox_policy: Option, - pub disable_response_storage: Option, pub model_provider: Option, pub config_profile: Option, pub codex_linux_sandbox_exe: Option, } impl Config { - /// Load configuration, optionally applying overrides (CLI flags). Merges - /// ~/.codex/config.toml, ~/.codex/instructions.md, embedded defaults, and - /// any values provided in `overrides` (highest precedence). - pub fn load_with_overrides(overrides: ConfigOverrides) -> std::io::Result { - // Resolve the directory that stores Codex state (e.g. ~/.codex or the - // value of $CODEX_HOME) so we can embed it into the resulting - // `Config` instance. - let codex_home = find_codex_home()?; - - let cfg: ConfigToml = ConfigToml::load_from_toml(&codex_home)?; - tracing::warn!("Config parsed from config.toml: {cfg:?}"); - - Self::load_from_base_config_with_overrides(cfg, overrides, codex_home) - } - /// Meant to be used exclusively for tests: `load_with_overrides()` should /// be used in all other cases. pub fn load_from_base_config_with_overrides( @@ -264,7 +328,6 @@ impl Config { cwd, approval_policy, sandbox_policy, - disable_response_storage, model_provider, config_profile: config_profile_key, codex_linux_sandbox_exe, @@ -356,8 +419,8 @@ impl Config { .unwrap_or_else(AskForApproval::default), sandbox_policy, shell_environment_policy, - disable_response_storage: disable_response_storage - .or(config_profile.disable_response_storage) + disable_response_storage: config_profile + .disable_response_storage .or(cfg.disable_response_storage) .unwrap_or(false), notify: cfg.notify, diff --git a/codex-rs/core/src/config_types.rs b/codex-rs/core/src/config_types.rs index 6696f76f0b..d89b09f267 100644 --- a/codex-rs/core/src/config_types.rs +++ b/codex-rs/core/src/config_types.rs @@ -89,7 +89,7 @@ pub struct Tui { } #[derive(Deserialize, Debug, Clone, PartialEq, Default)] - +#[serde(rename_all = "kebab-case")] pub enum ShellEnvironmentPolicyInherit { /// "Core" environment variables for the platform. On UNIX, this would /// include HOME, LOGNAME, PATH, SHELL, and USER, among others. diff --git a/codex-rs/exec/src/cli.rs b/codex-rs/exec/src/cli.rs index 4a3d493a89..1c2a9eb8aa 100644 --- a/codex-rs/exec/src/cli.rs +++ b/codex-rs/exec/src/cli.rs @@ -1,5 +1,6 @@ use clap::Parser; use clap::ValueEnum; +use codex_common::CliConfigOverrides; use codex_common::SandboxPermissionOption; use std::path::PathBuf; @@ -33,9 +34,8 @@ pub struct Cli { #[arg(long = "skip-git-repo-check", default_value_t = false)] pub skip_git_repo_check: bool, - /// Disable server‑side response storage (sends the full conversation context with every request) - #[arg(long = "disable-response-storage", default_value_t = false)] - pub disable_response_storage: bool, + #[clap(skip)] + pub config_overrides: CliConfigOverrides, /// Specifies color settings for use in the output. #[arg(long = "color", value_enum, default_value_t = Color::Auto)] diff --git a/codex-rs/exec/src/lib.rs b/codex-rs/exec/src/lib.rs index dbf01f025b..8c94fe5dc9 100644 --- a/codex-rs/exec/src/lib.rs +++ b/codex-rs/exec/src/lib.rs @@ -34,10 +34,10 @@ pub async fn run_main(cli: Cli, codex_linux_sandbox_exe: Option) -> any sandbox, cwd, skip_git_repo_check, - disable_response_storage, color, last_message_file, prompt, + config_overrides, } = cli; let (stdout_with_ansi, stderr_with_ansi) = match color { @@ -63,16 +63,20 @@ pub async fn run_main(cli: Cli, codex_linux_sandbox_exe: Option) -> any // the user for approval. approval_policy: Some(AskForApproval::Never), sandbox_policy, - disable_response_storage: if disable_response_storage { - Some(true) - } else { - None - }, cwd: cwd.map(|p| p.canonicalize().unwrap_or(p)), model_provider: None, codex_linux_sandbox_exe, }; - let config = Config::load_with_overrides(overrides)?; + // Parse `-c` overrides. + let cli_kv_overrides = match config_overrides.parse_overrides() { + Ok(v) => v, + Err(e) => { + eprintln!("Error parsing -c overrides: {e}"); + std::process::exit(1); + } + }; + + let config = Config::load_with_cli_overrides(cli_kv_overrides, overrides)?; // Print the effective configuration so users can see what Codex is using. print_config_summary(&config, stdout_with_ansi); diff --git a/codex-rs/exec/src/main.rs b/codex-rs/exec/src/main.rs index 17aa5377d2..3a8e1f9411 100644 --- a/codex-rs/exec/src/main.rs +++ b/codex-rs/exec/src/main.rs @@ -10,13 +10,30 @@ //! This allows us to ship a completely separate set of functionality as part //! of the `codex-exec` binary. use clap::Parser; +use codex_common::CliConfigOverrides; use codex_exec::Cli; use codex_exec::run_main; +#[derive(Parser, Debug)] +struct TopCli { + #[clap(flatten)] + config_overrides: CliConfigOverrides, + + #[clap(flatten)] + inner: Cli, +} + fn main() -> anyhow::Result<()> { codex_linux_sandbox::run_with_sandbox(|codex_linux_sandbox_exe| async move { - let cli = Cli::parse(); - run_main(cli, codex_linux_sandbox_exe).await?; + let top_cli = TopCli::parse(); + // Merge root-level overrides into inner CLI struct so downstream logic remains unchanged. + let mut inner = top_cli.inner; + inner + .config_overrides + .raw_overrides + .splice(0..0, top_cli.config_overrides.raw_overrides); + + run_main(inner, codex_linux_sandbox_exe).await?; Ok(()) }) } diff --git a/codex-rs/mcp-server/Cargo.toml b/codex-rs/mcp-server/Cargo.toml index 968222c943..c3f1115819 100644 --- a/codex-rs/mcp-server/Cargo.toml +++ b/codex-rs/mcp-server/Cargo.toml @@ -22,6 +22,7 @@ mcp-types = { path = "../mcp-types" } schemars = "0.8.22" serde = { version = "1", features = ["derive"] } serde_json = "1" +toml = "0.8" tracing = { version = "0.1.41", features = ["log"] } tracing-subscriber = { version = "0.3", features = ["fmt", "env-filter"] } tokio = { version = "1", features = [ diff --git a/codex-rs/mcp-server/src/codex_tool_config.rs b/codex-rs/mcp-server/src/codex_tool_config.rs index d04a5c80bc..03e7234449 100644 --- a/codex-rs/mcp-server/src/codex_tool_config.rs +++ b/codex-rs/mcp-server/src/codex_tool_config.rs @@ -1,15 +1,16 @@ //! Configuration object accepted by the `codex` MCP tool-call. -use std::path::PathBuf; - +use codex_core::protocol::AskForApproval; +use codex_core::protocol::SandboxPolicy; use mcp_types::Tool; use mcp_types::ToolInputSchema; use schemars::JsonSchema; use schemars::r#gen::SchemaSettings; use serde::Deserialize; +use std::collections::HashMap; +use std::path::PathBuf; -use codex_core::protocol::AskForApproval; -use codex_core::protocol::SandboxPolicy; +use crate::json_to_toml::json_to_toml; /// Client-supplied configuration for a `codex` tool-call. #[derive(Debug, Clone, Deserialize, JsonSchema)] @@ -41,12 +42,10 @@ pub(crate) struct CodexToolCallParam { #[serde(default, skip_serializing_if = "Option::is_none")] pub sandbox_permissions: Option>, - /// Disable server-side response storage. + /// Individual config settings that will override what is in + /// CODEX_HOME/config.toml. #[serde(default, skip_serializing_if = "Option::is_none")] - pub disable_response_storage: Option, - // Custom system instructions. - // #[serde(default, skip_serializing_if = "Option::is_none")] - // pub instructions: Option, + pub config: Option>, } // Create custom enums for use with `CodexToolCallApprovalPolicy` where we @@ -155,7 +154,7 @@ impl CodexToolCallParam { cwd, approval_policy, sandbox_permissions, - disable_response_storage, + config: cli_overrides, } = self; let sandbox_policy = sandbox_permissions.map(|perms| { SandboxPolicy::from(perms.into_iter().map(Into::into).collect::>()) @@ -168,12 +167,17 @@ impl CodexToolCallParam { cwd: cwd.map(PathBuf::from), approval_policy: approval_policy.map(Into::into), sandbox_policy, - disable_response_storage, model_provider: None, codex_linux_sandbox_exe, }; - let cfg = codex_core::config::Config::load_with_overrides(overrides)?; + let cli_overrides = cli_overrides + .unwrap_or_default() + .into_iter() + .map(|(k, v)| (k, json_to_toml(v))) + .collect(); + + let cfg = codex_core::config::Config::load_with_cli_overrides(cli_overrides, overrides)?; Ok((prompt, cfg)) } @@ -216,14 +220,15 @@ mod tests { ], "type": "string" }, + "config": { + "description": "Individual config settings that will override what is in CODEX_HOME/config.toml.", + "additionalProperties": true, + "type": "object" + }, "cwd": { "description": "Working directory for the session. If relative, it is resolved against the server process's current working directory.", "type": "string" }, - "disable-response-storage": { - "description": "Disable server-side response storage.", - "type": "boolean" - }, "model": { "description": "Optional override for the model name (e.g. \"o3\", \"o4-mini\")", "type": "string" diff --git a/codex-rs/mcp-server/src/json_to_toml.rs b/codex-rs/mcp-server/src/json_to_toml.rs new file mode 100644 index 0000000000..ae33382a1d --- /dev/null +++ b/codex-rs/mcp-server/src/json_to_toml.rs @@ -0,0 +1,84 @@ +use serde_json::Value as JsonValue; +use toml::Value as TomlValue; + +/// Convert a `serde_json::Value` into a semantically equivalent `toml::Value`. +pub(crate) fn json_to_toml(v: JsonValue) -> TomlValue { + match v { + JsonValue::Null => TomlValue::String(String::new()), + JsonValue::Bool(b) => TomlValue::Boolean(b), + JsonValue::Number(n) => { + if let Some(i) = n.as_i64() { + TomlValue::Integer(i) + } else if let Some(f) = n.as_f64() { + TomlValue::Float(f) + } else { + TomlValue::String(n.to_string()) + } + } + JsonValue::String(s) => TomlValue::String(s), + JsonValue::Array(arr) => TomlValue::Array(arr.into_iter().map(json_to_toml).collect()), + JsonValue::Object(map) => { + let tbl = map + .into_iter() + .map(|(k, v)| (k, json_to_toml(v))) + .collect::(); + TomlValue::Table(tbl) + } + } +} + +#[cfg(test)] +#[allow(clippy::unwrap_used)] +mod tests { + use super::*; + use pretty_assertions::assert_eq; + use serde_json::json; + + #[test] + fn json_number_to_toml() { + let json_value = json!(123); + assert_eq!(TomlValue::Integer(123), json_to_toml(json_value)); + } + + #[test] + fn json_array_to_toml() { + let json_value = json!([true, 1]); + assert_eq!( + TomlValue::Array(vec![TomlValue::Boolean(true), TomlValue::Integer(1)]), + json_to_toml(json_value) + ); + } + + #[test] + fn json_bool_to_toml() { + let json_value = json!(false); + assert_eq!(TomlValue::Boolean(false), json_to_toml(json_value)); + } + + #[test] + fn json_float_to_toml() { + let json_value = json!(1.25); + assert_eq!(TomlValue::Float(1.25), json_to_toml(json_value)); + } + + #[test] + fn json_null_to_toml() { + let json_value = serde_json::Value::Null; + assert_eq!(TomlValue::String(String::new()), json_to_toml(json_value)); + } + + #[test] + fn json_object_nested() { + let json_value = json!({ "outer": { "inner": 2 } }); + let expected = { + let mut inner = toml::value::Table::new(); + inner.insert("inner".into(), TomlValue::Integer(2)); + + let mut outer = toml::value::Table::new(); + outer.insert("outer".into(), TomlValue::Table(inner)); + TomlValue::Table(outer) + }; + + assert_eq!(json_to_toml(json_value), expected); + } +} diff --git a/codex-rs/mcp-server/src/lib.rs b/codex-rs/mcp-server/src/lib.rs index 0f29eb7826..b2a7797fe6 100644 --- a/codex-rs/mcp-server/src/lib.rs +++ b/codex-rs/mcp-server/src/lib.rs @@ -16,6 +16,7 @@ use tracing::info; mod codex_tool_config; mod codex_tool_runner; +mod json_to_toml; mod message_processor; use crate::message_processor::MessageProcessor; diff --git a/codex-rs/tui/src/cli.rs b/codex-rs/tui/src/cli.rs index f077d26743..4abd684144 100644 --- a/codex-rs/tui/src/cli.rs +++ b/codex-rs/tui/src/cli.rs @@ -1,5 +1,6 @@ use clap::Parser; use codex_common::ApprovalModeCliArg; +use codex_common::CliConfigOverrides; use codex_common::SandboxPermissionOption; use std::path::PathBuf; @@ -40,7 +41,6 @@ pub struct Cli { #[arg(long = "skip-git-repo-check", default_value_t = false)] pub skip_git_repo_check: bool, - /// Disable server‑side response storage (sends the full conversation context with every request) - #[arg(long = "disable-response-storage", default_value_t = false)] - pub disable_response_storage: bool, + #[clap(skip)] + pub config_overrides: CliConfigOverrides, } diff --git a/codex-rs/tui/src/lib.rs b/codex-rs/tui/src/lib.rs index 4ab68724aa..1ddd79cf1a 100644 --- a/codex-rs/tui/src/lib.rs +++ b/codex-rs/tui/src/lib.rs @@ -54,18 +54,23 @@ pub fn run_main(cli: Cli, codex_linux_sandbox_exe: Option) -> std::io:: model: cli.model.clone(), approval_policy, sandbox_policy, - disable_response_storage: if cli.disable_response_storage { - Some(true) - } else { - None - }, cwd: cli.cwd.clone().map(|p| p.canonicalize().unwrap_or(p)), model_provider: None, config_profile: cli.config_profile.clone(), codex_linux_sandbox_exe, }; + // Parse `-c` overrides from the CLI. + let cli_kv_overrides = match cli.config_overrides.parse_overrides() { + Ok(v) => v, + #[allow(clippy::print_stderr)] + Err(e) => { + eprintln!("Error parsing -c overrides: {e}"); + std::process::exit(1); + } + }; + #[allow(clippy::print_stderr)] - match Config::load_with_overrides(overrides) { + match Config::load_with_cli_overrides(cli_kv_overrides, overrides) { Ok(config) => config, Err(err) => { eprintln!("Error loading configuration: {err}"); diff --git a/codex-rs/tui/src/main.rs b/codex-rs/tui/src/main.rs index 7e55f2af5d..7fcc944504 100644 --- a/codex-rs/tui/src/main.rs +++ b/codex-rs/tui/src/main.rs @@ -1,11 +1,26 @@ use clap::Parser; +use codex_common::CliConfigOverrides; use codex_tui::Cli; use codex_tui::run_main; +#[derive(Parser, Debug)] +struct TopCli { + #[clap(flatten)] + config_overrides: CliConfigOverrides, + + #[clap(flatten)] + inner: Cli, +} + fn main() -> anyhow::Result<()> { codex_linux_sandbox::run_with_sandbox(|codex_linux_sandbox_exe| async move { - let cli = Cli::parse(); - run_main(cli, codex_linux_sandbox_exe)?; + let top_cli = TopCli::parse(); + let mut inner = top_cli.inner; + inner + .config_overrides + .raw_overrides + .splice(0..0, top_cli.config_overrides.raw_overrides); + run_main(inner, codex_linux_sandbox_exe)?; Ok(()) }) } From ae1a83f095baf45eb3ba43c5dc947ec7bd7ce9b8 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Wed, 28 May 2025 14:03:19 -0700 Subject: [PATCH 02/41] feat: introduce CellWidget trait (#1148) The motivation behind this PR is to make it so a `HistoryCell` is more like a `WidgetRef` that knows how to render itself into a `Rect` so that it can be backed by something other than a `Vec`. Because a `HistoryCell` is intended to appear in a scrollable list, we want to ensure the stack of cells can be scrolled one `Line` at a time even if the `HistoryCell` is not backed by a `Vec` itself. To this end, we introduce the `CellWidget` trait whose key method is: ``` fn render_window(&self, first_visible_line: usize, area: Rect, buf: &mut Buffer); ``` The `first_visible_line` param is what differs from `WidgetRef::render_ref()`, as a `CellWidget` needs to know the offset into its "full view" at which it should start rendering. The bookkeeping in `ConversationHistoryWidget` has been updated accordingly to ensure each `CellWidget` in the history is rendered appropriately. --- codex-rs/tui/src/cell_widget.rs | 20 +++ .../tui/src/conversation_history_widget.rs | 134 +++++++---------- codex-rs/tui/src/history_cell.rs | 140 ++++++++++++------ codex-rs/tui/src/lib.rs | 2 + codex-rs/tui/src/text_block.rs | 32 ++++ 5 files changed, 203 insertions(+), 125 deletions(-) create mode 100644 codex-rs/tui/src/cell_widget.rs create mode 100644 codex-rs/tui/src/text_block.rs diff --git a/codex-rs/tui/src/cell_widget.rs b/codex-rs/tui/src/cell_widget.rs new file mode 100644 index 0000000000..8acdc0553a --- /dev/null +++ b/codex-rs/tui/src/cell_widget.rs @@ -0,0 +1,20 @@ +use ratatui::prelude::*; + +/// Trait implemented by every type that can live inside the conversation +/// history list. It provides two primitives that the parent scroll-view +/// needs: how *tall* the widget is at a given width and how to render an +/// arbitrary contiguous *window* of that widget. +/// +/// The `first_visible_line` argument to [`render_window`] allows partial +/// rendering when the top of the widget is scrolled off-screen. The caller +/// guarantees that `first_visible_line + area.height as usize` never exceeds +/// the total height previously returned by [`height`]. +pub(crate) trait CellWidget { + /// Total height measured in wrapped terminal lines when drawn with the + /// given *content* width (no scrollbar column included). + fn height(&self, width: u16) -> usize; + + /// Render a *window* that starts `first_visible_line` lines below the top + /// of the widget. The window’s size is given by `area`. + fn render_window(&self, first_visible_line: usize, area: Rect, buf: &mut Buffer); +} diff --git a/codex-rs/tui/src/conversation_history_widget.rs b/codex-rs/tui/src/conversation_history_widget.rs index 83d5ebc496..d69f4db88e 100644 --- a/codex-rs/tui/src/conversation_history_widget.rs +++ b/codex-rs/tui/src/conversation_history_widget.rs @@ -1,3 +1,4 @@ +use crate::cell_widget::CellWidget; use crate::history_cell::CommandOutput; use crate::history_cell::HistoryCell; use crate::history_cell::PatchEventType; @@ -236,11 +237,7 @@ impl ConversationHistoryWidget { fn add_to_history(&mut self, cell: HistoryCell) { let width = self.cached_width.get(); - let count = if width > 0 { - wrapped_line_count_for_cell(&cell, width) - } else { - 0 - }; + let count = if width > 0 { cell.height(width) } else { 0 }; self.entries.push(Entry { cell, @@ -284,9 +281,7 @@ impl ConversationHistoryWidget { // Update cached line count. if width > 0 { - entry - .line_count - .set(wrapped_line_count_for_cell(cell, width)); + entry.line_count.set(cell.height(width)); } break; } @@ -328,9 +323,7 @@ impl ConversationHistoryWidget { entry.cell = completed; if width > 0 { - entry - .line_count - .set(wrapped_line_count_for_cell(&entry.cell, width)); + entry.line_count.set(entry.cell.height(width)); } break; @@ -378,7 +371,7 @@ impl WidgetRef for ConversationHistoryWidget { let mut num_lines: usize = 0; for entry in &self.entries { - let count = wrapped_line_count_for_cell(&entry.cell, effective_width); + let count = entry.cell.height(effective_width); num_lines += count; entry.line_count.set(count); } @@ -398,78 +391,68 @@ impl WidgetRef for ConversationHistoryWidget { }; // ------------------------------------------------------------------ - // Build a *window* into the history so we only clone the `Line`s that - // may actually be visible in this frame. We still hand the slice off - // to a `Paragraph` with an additional scroll offset to avoid slicing - // inside a wrapped line (we don’t have per-subline granularity). + // Render order: + // 1. Clear full widget area (avoid artifacts from prior frame). + // 2. Draw the surrounding Block (border and title). + // 3. Render *each* visible HistoryCell into its own sub-Rect while + // respecting partial visibility at the top and bottom. + // 4. Draw the scrollbar track / thumb in the reserved column. // ------------------------------------------------------------------ - // Find the first entry that intersects the current scroll position. - let mut cumulative = 0usize; - let mut first_idx = 0usize; - for (idx, entry) in self.entries.iter().enumerate() { - let next = cumulative + entry.line_count.get(); - if next > scroll_pos { - first_idx = idx; - break; - } - cumulative = next; - } + // Clear entire widget area first. + Clear.render(area, buf); - let offset_into_first = scroll_pos - cumulative; + // Draw border + title. + block.render(area, buf); + + // ------------------------------------------------------------------ + // Calculate which cells are visible for the current scroll position + // and paint them one by one. + // ------------------------------------------------------------------ + + let mut y_cursor = inner.y; // first line inside viewport + let mut remaining_height = inner.height as usize; + let mut lines_to_skip = scroll_pos; // number of wrapped lines to skip (above viewport) - // Collect enough raw lines from `first_idx` onward to cover the - // viewport. We may fetch *slightly* more than necessary (whole cells) - // but never the entire history. - let mut collected_wrapped = 0usize; - let mut visible_lines: Vec> = Vec::new(); + for entry in &self.entries { + let cell_height = entry.line_count.get(); - for entry in &self.entries[first_idx..] { - visible_lines.extend(entry.cell.lines().iter().cloned()); - collected_wrapped += entry.line_count.get(); - if collected_wrapped >= offset_into_first + viewport_height { - break; + // Completely above viewport? Skip whole cell. + if lines_to_skip >= cell_height { + lines_to_skip -= cell_height; + continue; } - } - // Build the Paragraph with wrapping enabled so long lines are not - // clipped. Apply vertical scroll so that `offset_into_first` wrapped - // lines are hidden at the top. - // ------------------------------------------------------------------ - // Render order: - // 1. Clear the whole widget area so we do not leave behind any glyphs - // from the previous frame. - // 2. Draw the surrounding Block (border and title). - // 3. Draw the Paragraph inside the Block, **leaving the right-most - // column free** for the scrollbar. - // 4. Finally draw the scrollbar (if needed). - // ------------------------------------------------------------------ + // Determine how much of this cell is visible. + let visible_height = (cell_height - lines_to_skip).min(remaining_height); - // Clear the widget area to avoid visual artifacts from previous frames. - Clear.render(area, buf); + if visible_height == 0 { + break; // no space left + } - // Draw the outer border and title first so the Paragraph does not - // overwrite it. - block.render(area, buf); + let cell_rect = Rect { + x: inner.x, + y: y_cursor, + width: effective_width, + height: visible_height as u16, + }; - // Area available for text after accounting for the scrollbar. - let text_area = Rect { - x: inner.x, - y: inner.y, - width: effective_width, - height: inner.height, - }; + entry.cell.render_window(lines_to_skip, cell_rect, buf); - let paragraph = Paragraph::new(visible_lines) - .wrap(wrap_cfg()) - .scroll((offset_into_first as u16, 0)); + // Advance cursor inside viewport. + y_cursor += visible_height as u16; + remaining_height -= visible_height; - paragraph.render(text_area, buf); + // After the first (possibly partially skipped) cell, we no longer + // need to skip lines at the top. + lines_to_skip = 0; - // Always render a scrollbar *track* so that the reserved column is - // visually filled, even when the content fits within the viewport. - // We only draw the *thumb* when the content actually overflows. + if remaining_height == 0 { + break; // viewport filled + } + } + // Always render a scrollbar *track* so the reserved column is filled. let overflow = num_lines.saturating_sub(viewport_height); let mut scroll_state = ScrollbarState::default() @@ -521,15 +504,6 @@ impl WidgetRef for ConversationHistoryWidget { /// Common [`Wrap`] configuration used for both measurement and rendering so /// they stay in sync. #[inline] -const fn wrap_cfg() -> ratatui::widgets::Wrap { +pub(crate) const fn wrap_cfg() -> ratatui::widgets::Wrap { ratatui::widgets::Wrap { trim: false } } - -/// Returns the wrapped line count for `cell` at the given `width` using the -/// same wrapping rules that `ConversationHistoryWidget` uses during -/// rendering. -fn wrapped_line_count_for_cell(cell: &HistoryCell, width: u16) -> usize { - Paragraph::new(cell.lines().clone()) - .wrap(wrap_cfg()) - .line_count(width) -} diff --git a/codex-rs/tui/src/history_cell.rs b/codex-rs/tui/src/history_cell.rs index fab9432724..c2938f4b85 100644 --- a/codex-rs/tui/src/history_cell.rs +++ b/codex-rs/tui/src/history_cell.rs @@ -9,6 +9,9 @@ use ratatui::style::Modifier; use ratatui::style::Style; use ratatui::text::Line as RtLine; use ratatui::text::Span as RtSpan; + +use crate::cell_widget::CellWidget; +use crate::text_block::TextBlock; use std::collections::HashMap; use std::path::PathBuf; use std::time::Duration; @@ -34,16 +37,16 @@ pub(crate) enum PatchEventType { /// scrollable list. pub(crate) enum HistoryCell { /// Welcome message. - WelcomeMessage { lines: Vec> }, + WelcomeMessage { view: TextBlock }, /// Message from the user. - UserPrompt { lines: Vec> }, + UserPrompt { view: TextBlock }, /// Message from the agent. - AgentMessage { lines: Vec> }, + AgentMessage { view: TextBlock }, /// Reasoning event from the agent. - AgentReasoning { lines: Vec> }, + AgentReasoning { view: TextBlock }, /// An exec tool call that has not finished yet. ActiveExecCommand { @@ -51,11 +54,11 @@ pub(crate) enum HistoryCell { /// The shell command, escaped and formatted. command: String, start: Instant, - lines: Vec>, + view: TextBlock, }, /// Completed exec tool call. - CompletedExecCommand { lines: Vec> }, + CompletedExecCommand { view: TextBlock }, /// An MCP tool call that has not finished yet. ActiveMcpToolCall { @@ -67,29 +70,25 @@ pub(crate) enum HistoryCell { /// exact same text without re-formatting. invocation: String, start: Instant, - lines: Vec>, + view: TextBlock, }, /// Completed MCP tool call. - CompletedMcpToolCall { lines: Vec> }, + CompletedMcpToolCall { view: TextBlock }, - /// Background event - BackgroundEvent { lines: Vec> }, + /// Background event. + BackgroundEvent { view: TextBlock }, /// Error event from the backend. - ErrorEvent { lines: Vec> }, + ErrorEvent { view: TextBlock }, - /// Info describing the newly‑initialized session. - SessionInfo { lines: Vec> }, + /// Info describing the newly-initialized session. + SessionInfo { view: TextBlock }, /// A pending code patch that is awaiting user approval. Mirrors the /// behaviour of `ActiveExecCommand` so the user sees *what* patch the /// model wants to apply before being prompted to approve or deny it. - PendingPatch { - /// Identifier so that a future `PatchApplyEnd` can update the entry - /// with the final status (not yet implemented). - lines: Vec>, - }, + PendingPatch { view: TextBlock }, } const TOOL_CALL_MAX_LINES: usize = 5; @@ -132,9 +131,13 @@ impl HistoryCell { lines.push(Line::from(vec![format!("{key}: ").bold(), value.into()])); } lines.push(Line::from("")); - HistoryCell::WelcomeMessage { lines } + HistoryCell::WelcomeMessage { + view: TextBlock::new(lines), + } } else if config.model == model { - HistoryCell::SessionInfo { lines: vec![] } + HistoryCell::SessionInfo { + view: TextBlock::new(Vec::new()), + } } else { let lines = vec![ Line::from("model changed:".magenta().bold()), @@ -142,7 +145,9 @@ impl HistoryCell { Line::from(format!("used: {}", model)), Line::from(""), ]; - HistoryCell::SessionInfo { lines } + HistoryCell::SessionInfo { + view: TextBlock::new(lines), + } } } @@ -152,7 +157,9 @@ impl HistoryCell { lines.extend(message.lines().map(|l| Line::from(l.to_string()))); lines.push(Line::from("")); - HistoryCell::UserPrompt { lines } + HistoryCell::UserPrompt { + view: TextBlock::new(lines), + } } pub(crate) fn new_agent_message(config: &Config, message: String) -> Self { @@ -161,7 +168,9 @@ impl HistoryCell { append_markdown(&message, &mut lines, config); lines.push(Line::from("")); - HistoryCell::AgentMessage { lines } + HistoryCell::AgentMessage { + view: TextBlock::new(lines), + } } pub(crate) fn new_agent_reasoning(config: &Config, text: String) -> Self { @@ -170,7 +179,9 @@ impl HistoryCell { append_markdown(&text, &mut lines, config); lines.push(Line::from("")); - HistoryCell::AgentReasoning { lines } + HistoryCell::AgentReasoning { + view: TextBlock::new(lines), + } } pub(crate) fn new_active_exec_command(call_id: String, command: Vec) -> Self { @@ -187,7 +198,7 @@ impl HistoryCell { call_id, command: command_escaped, start, - lines, + view: TextBlock::new(lines), } } @@ -226,7 +237,9 @@ impl HistoryCell { } lines.push(Line::from("")); - HistoryCell::CompletedExecCommand { lines } + HistoryCell::CompletedExecCommand { + view: TextBlock::new(lines), + } } pub(crate) fn new_active_mcp_tool_call( @@ -267,7 +280,7 @@ impl HistoryCell { fq_tool_name, invocation, start, - lines, + view: TextBlock::new(lines), } } @@ -304,7 +317,9 @@ impl HistoryCell { lines.push(Line::from("")); - HistoryCell::CompletedMcpToolCall { lines } + HistoryCell::CompletedMcpToolCall { + view: TextBlock::new(lines), + } } pub(crate) fn new_background_event(message: String) -> Self { @@ -312,7 +327,9 @@ impl HistoryCell { lines.push(Line::from("event".dim())); lines.extend(message.lines().map(|l| Line::from(l.to_string()).dim())); lines.push(Line::from("")); - HistoryCell::BackgroundEvent { lines } + HistoryCell::BackgroundEvent { + view: TextBlock::new(lines), + } } pub(crate) fn new_error_event(message: String) -> Self { @@ -320,7 +337,9 @@ impl HistoryCell { vec!["ERROR: ".red().bold(), message.into()].into(), "".into(), ]; - HistoryCell::ErrorEvent { lines } + HistoryCell::ErrorEvent { + view: TextBlock::new(lines), + } } /// Create a new `PendingPatch` cell that lists the file‑level summary of @@ -339,7 +358,9 @@ impl HistoryCell { auto_approved: false, } => { let lines = vec![Line::from("patch applied".magenta().bold())]; - return Self::PendingPatch { lines }; + return Self::PendingPatch { + view: TextBlock::new(lines), + }; } }; @@ -380,23 +401,52 @@ impl HistoryCell { lines.push(Line::from("")); - HistoryCell::PendingPatch { lines } + HistoryCell::PendingPatch { + view: TextBlock::new(lines), + } + } +} + +// --------------------------------------------------------------------------- +// `CellWidget` implementation – most variants delegate to their internal +// `TextBlock`. Variants that need custom painting can add their own logic in +// the match arms. +// --------------------------------------------------------------------------- + +impl CellWidget for HistoryCell { + fn height(&self, width: u16) -> usize { + match self { + HistoryCell::WelcomeMessage { view } + | HistoryCell::UserPrompt { view } + | HistoryCell::AgentMessage { view } + | HistoryCell::AgentReasoning { view } + | HistoryCell::BackgroundEvent { view } + | HistoryCell::ErrorEvent { view } + | HistoryCell::SessionInfo { view } + | HistoryCell::CompletedExecCommand { view } + | HistoryCell::CompletedMcpToolCall { view } + | HistoryCell::PendingPatch { view } + | HistoryCell::ActiveExecCommand { view, .. } + | HistoryCell::ActiveMcpToolCall { view, .. } => view.height(width), + } } - pub(crate) fn lines(&self) -> &Vec> { + fn render_window(&self, first_visible_line: usize, area: Rect, buf: &mut Buffer) { match self { - HistoryCell::WelcomeMessage { lines, .. } - | HistoryCell::UserPrompt { lines, .. } - | HistoryCell::AgentMessage { lines, .. } - | HistoryCell::AgentReasoning { lines, .. } - | HistoryCell::BackgroundEvent { lines, .. } - | HistoryCell::ErrorEvent { lines, .. } - | HistoryCell::SessionInfo { lines, .. } - | HistoryCell::ActiveExecCommand { lines, .. } - | HistoryCell::CompletedExecCommand { lines, .. } - | HistoryCell::ActiveMcpToolCall { lines, .. } - | HistoryCell::CompletedMcpToolCall { lines, .. } - | HistoryCell::PendingPatch { lines, .. } => lines, + HistoryCell::WelcomeMessage { view } + | HistoryCell::UserPrompt { view } + | HistoryCell::AgentMessage { view } + | HistoryCell::AgentReasoning { view } + | HistoryCell::BackgroundEvent { view } + | HistoryCell::ErrorEvent { view } + | HistoryCell::SessionInfo { view } + | HistoryCell::CompletedExecCommand { view } + | HistoryCell::CompletedMcpToolCall { view } + | HistoryCell::PendingPatch { view } + | HistoryCell::ActiveExecCommand { view, .. } + | HistoryCell::ActiveMcpToolCall { view, .. } => { + view.render_window(first_visible_line, area, buf) + } } } } diff --git a/codex-rs/tui/src/lib.rs b/codex-rs/tui/src/lib.rs index 1ddd79cf1a..df85673ef1 100644 --- a/codex-rs/tui/src/lib.rs +++ b/codex-rs/tui/src/lib.rs @@ -19,6 +19,7 @@ mod app; mod app_event; mod app_event_sender; mod bottom_pane; +mod cell_widget; mod chatwidget; mod citation_regex; mod cli; @@ -32,6 +33,7 @@ mod mouse_capture; mod scroll_event_helper; mod slash_command; mod status_indicator_widget; +mod text_block; mod tui; mod user_approval_widget; diff --git a/codex-rs/tui/src/text_block.rs b/codex-rs/tui/src/text_block.rs new file mode 100644 index 0000000000..2c68d90f11 --- /dev/null +++ b/codex-rs/tui/src/text_block.rs @@ -0,0 +1,32 @@ +use crate::cell_widget::CellWidget; +use ratatui::prelude::*; + +/// A simple widget that just displays a list of `Line`s via a `Paragraph`. +/// This is the default rendering backend for most `HistoryCell` variants. +#[derive(Clone)] +pub(crate) struct TextBlock { + pub(crate) lines: Vec>, +} + +impl TextBlock { + pub(crate) fn new(lines: Vec>) -> Self { + Self { lines } + } +} + +impl CellWidget for TextBlock { + fn height(&self, width: u16) -> usize { + // Use the same wrapping configuration as ConversationHistoryWidget so + // measurement stays in sync with rendering. + ratatui::widgets::Paragraph::new(self.lines.clone()) + .wrap(crate::conversation_history_widget::wrap_cfg()) + .line_count(width) + } + + fn render_window(&self, first_visible_line: usize, area: Rect, buf: &mut Buffer) { + ratatui::widgets::Paragraph::new(self.lines.clone()) + .wrap(crate::conversation_history_widget::wrap_cfg()) + .scroll((first_visible_line as u16, 0)) + .render(area, buf); + } +} From 392fdd7db6e750cdefa158a62324fab54ca6dfe8 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Wed, 28 May 2025 17:10:06 -0700 Subject: [PATCH 03/41] fix: honor RUST_LOG in mcp-client CLI and default to DEBUG (#1149) We had `debug!()` logging statements already, but they weren't being printed because `tracing_subscriber` was not set up. --- codex-rs/mcp-client/src/main.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/codex-rs/mcp-client/src/main.rs b/codex-rs/mcp-client/src/main.rs index af4b05098d..518383d1ea 100644 --- a/codex-rs/mcp-client/src/main.rs +++ b/codex-rs/mcp-client/src/main.rs @@ -20,9 +20,22 @@ use mcp_types::Implementation; use mcp_types::InitializeRequestParams; use mcp_types::ListToolsRequestParams; use mcp_types::MCP_SCHEMA_VERSION; +use tracing_subscriber::EnvFilter; #[tokio::main] async fn main() -> Result<()> { + let default_level = "debug"; + let _ = tracing_subscriber::fmt() + // Fallback to the `default_level` log filter if the environment + // variable is not set _or_ contains an invalid value + .with_env_filter( + EnvFilter::try_from_default_env() + .or_else(|_| EnvFilter::try_new(default_level)) + .unwrap_or_else(|_| EnvFilter::new(default_level)), + ) + .with_writer(std::io::stderr) + .try_init(); + // Collect command-line arguments excluding the program name itself. let mut args: Vec = std::env::args().skip(1).collect(); From 25a9949c49194d5a64de54a11bcc5b4724ac9bd5 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Wed, 28 May 2025 17:17:21 -0700 Subject: [PATCH 04/41] fix: ensure inputSchema for MCP tool always has "properties" field when talking to OpenAI (#1150) As noted in the comment introduced in this PR, this is analogous to the issue reported in https://github.com/openai/openai-agents-python/issues/449. This seems to work now. --- codex-rs/core/src/client.rs | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/codex-rs/core/src/client.rs b/codex-rs/core/src/client.rs index 57534e2f9a..72ce845fc8 100644 --- a/codex-rs/core/src/client.rs +++ b/codex-rs/core/src/client.rs @@ -280,12 +280,26 @@ fn mcp_tool_to_openai_tool( fully_qualified_name: String, tool: mcp_types::Tool, ) -> serde_json::Value { + let mcp_types::Tool { + description, + mut input_schema, + .. + } = tool; + + // OpenAI models mandate the "properties" field in the schema. The Agents + // SDK fixed this by inserting an empty object for "properties" if it is not + // already present https://github.com/openai/openai-agents-python/issues/449 + // so here we do the same. + if input_schema.properties.is_none() { + input_schema.properties = Some(serde_json::Value::Object(serde_json::Map::new())); + } + // TODO(mbolin): Change the contract of this function to return // ResponsesApiTool. json!({ "name": fully_qualified_name, - "description": tool.description, - "parameters": tool.input_schema, + "description": description, + "parameters": input_schema, "type": "function", }) } From a768a6a41d02248ee795c941af88e283f8c81c9d Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Wed, 28 May 2025 19:03:17 -0700 Subject: [PATCH 05/41] fix: introduce ResponseInputItem::McpToolCallOutput variant (#1151) The output of an MCP server tool call can be one of several types, but to date, we treated all outputs as text by showing the serialized JSON as the "tool output" in Codex: https://github.com/openai/codex/blob/25a9949c49194d5a64de54a11bcc5b4724ac9bd5/codex-rs/mcp-types/src/lib.rs#L96-L101 This PR adds support for the `ImageContent` variant so we can now display an image output from an MCP tool call. In making this change, we introduce a new `ResponseInputItem::McpToolCallOutput` variant so that we can work with the `mcp_types::CallToolResult` directly when the function call is made to an MCP server. Though arguably the more significant change is the introduction of `HistoryCell::CompletedMcpToolCallWithImageOutput`, which is a cell that uses `ratatui_image` to render an image into the terminal. To support this, we introduce `ImageRenderCache`, cache a `ratatui_image::picker::Picker`, and `ensure_image_cache()` to cache the appropriate scaled image data and dimensions based on the current terminal size. To test, I created a minimal `package.json`: ```json { "name": "kitty-mcp", "version": "1.0.0", "type": "module", "description": "MCP that returns image of kitty", "main": "index.js", "dependencies": { "@modelcontextprotocol/sdk": "^1.12.0" } } ``` with the following `index.js` to define the MCP server: ```js #!/usr/bin/env node import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js"; import { readFile } from "node:fs/promises"; import { join } from "node:path"; const IMAGE_URI = "image://Ada.png"; const server = new McpServer({ name: "Demo", version: "1.0.0", }); server.tool( "get-cat-image", "If you need a cat image, this tool will provide one.", async () => ({ content: [ { type: "image", data: await getAdaPngBase64(), mimeType: "image/png" }, ], }) ); server.resource("Ada the Cat", IMAGE_URI, async (uri) => { const base64Image = await getAdaPngBase64(); return { contents: [ { uri: uri.href, mimeType: "image/png", blob: base64Image, }, ], }; }); async function getAdaPngBase64() { const __dirname = new URL(".", import.meta.url).pathname; // From https://github.com/benjajaja/ratatui-image/blob/9705ce2c59ec669abbce2924cbfd1f5ae22c9860/assets/Ada.png const filePath = join(__dirname, "Ada.png"); const imageData = await readFile(filePath); const base64Image = imageData.toString("base64"); return base64Image; } const transport = new StdioServerTransport(); await server.connect(transport); ``` With the local changes from this PR, I added the following to my `config.toml`: ```toml [mcp_servers.kitty] command = "node" args = ["/Users/mbolin/code/kitty-mcp/index.js"] ``` Running the TUI from source: ``` cargo run --bin codex -- --model o3 'I need a picture of a cat' ``` I get: image Now, that said, I have only tested in iTerm and there is definitely some funny business with getting an accurate character-to-pixel ratio (sometimes the `CompletedMcpToolCallWithImageOutput` thinks it needs 10 rows to render instead of 4), so there is still work to be done here. --- codex-rs/Cargo.lock | 654 +++++++++++++++++- codex-rs/core/src/mcp_tool_call.rs | 47 +- codex-rs/core/src/models.rs | 18 + codex-rs/core/src/protocol.rs | 13 +- codex-rs/exec/src/event_processor.rs | 14 +- codex-rs/tui/Cargo.toml | 3 + codex-rs/tui/src/chatwidget.rs | 8 +- .../tui/src/conversation_history_widget.rs | 11 +- codex-rs/tui/src/history_cell.rs | 255 ++++++- 9 files changed, 936 insertions(+), 87 deletions(-) diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 8f1762cac6..97a90c1520 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -54,6 +54,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "aligned-vec" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4aa90d7ce82d4be67b64039a3d588d38dbcc6736577de4a847025ce5b0c468d1" + [[package]] name = "allocative" version = "0.3.4" @@ -177,6 +183,29 @@ version = "1.0.98" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e16d2d3311acee920a9eb8d33b8cbc1787ce4a264e85f964c2404b969bdcd487" +[[package]] +name = "arbitrary" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dde20b3d026af13f561bdd0f15edf01fc734f0dafcedbaf42bba506a9517f223" + +[[package]] +name = "arg_enum_proc_macro" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ae92a5119aa49cdbcf6b9f893fe4e1d98b04ccbf82ee0584ad948a44a734dea" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + +[[package]] +name = "arrayvec" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" + [[package]] name = "ascii-canvas" version = "3.0.0" @@ -247,6 +276,29 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" +[[package]] +name = "av1-grain" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f3efb2ca85bc610acfa917b5aaa36f3fcbebed5b3182d7f877b02531c4b80c8" +dependencies = [ + "anyhow", + "arrayvec", + "log", + "nom", + "num-rational", + "v_frame", +] + +[[package]] +name = "avif-serialize" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98922d6a4cfbcb08820c69d8eeccc05bb1f29bfa06b4f5b1dbfe9a868bd7608e" +dependencies = [ + "arrayvec", +] + [[package]] name = "backtrace" version = "0.3.71" @@ -304,6 +356,12 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" +[[package]] +name = "bit_field" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc827186963e592360843fb5ba4b973e145841266c1357f7180c43526f2e5b61" + [[package]] name = "bitflags" version = "1.3.2" @@ -316,6 +374,12 @@ version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c8214115b7bf84099f1309324e63141d4c5d7cc26862f97a0a857dbefe165bd" +[[package]] +name = "bitstream-io" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6099cdc01846bc367c4e7dd630dc5966dccf36b652fae7a74e17b640411a91b2" + [[package]] name = "bstr" version = "1.12.0" @@ -327,6 +391,12 @@ dependencies = [ "serde", ] +[[package]] +name = "built" +version = "0.7.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56ed6191a7e78c36abdb16ab65341eefd73d64d303fffccdbb00d51e4205967b" + [[package]] name = "bumpalo" version = "3.17.0" @@ -339,12 +409,24 @@ version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5ce89b21cab1437276d2650d57e971f9d548a2d9037cc231abdc0562b97498ce" +[[package]] +name = "bytemuck" +version = "1.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9134a6ef01ce4b366b50689c94f82c14bc72bc5d0386829828a2e2752ef7958c" + [[package]] name = "byteorder" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" +[[package]] +name = "byteorder-lite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495" + [[package]] name = "bytes" version = "1.10.1" @@ -372,9 +454,21 @@ version = "1.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e3a13707ac958681c13b39b458c073d0d9bc8a22cb1b2f4c8e55eb72c13f362" dependencies = [ + "jobserver", + "libc", "shlex", ] +[[package]] +name = "cfg-expr" +version = "0.15.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d067ad48b8650848b989a59a86c6c36a995d02d2bf778d45c3c5d57bc2718f02" +dependencies = [ + "smallvec", + "target-lexicon", +] + [[package]] name = "cfg-if" version = "1.0.0" @@ -536,7 +630,7 @@ dependencies = [ "path-absolutize", "predicates", "pretty_assertions", - "rand", + "rand 0.9.1", "reqwest", "seccompiler", "serde", @@ -646,6 +740,7 @@ name = "codex-tui" version = "0.0.0" dependencies = [ "anyhow", + "base64 0.22.1", "clap", "codex-ansi-escape", "codex-common", @@ -653,11 +748,13 @@ dependencies = [ "codex-linux-sandbox", "color-eyre", "crossterm", + "image", "lazy_static", "mcp-types", "path-clean", "pretty_assertions", "ratatui", + "ratatui-image", "regex", "serde_json", "shlex", @@ -700,6 +797,12 @@ dependencies = [ "tracing-error", ] +[[package]] +name = "color_quant" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" + [[package]] name = "colorchoice" version = "1.0.3" @@ -772,6 +875,25 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-utils" version = "0.8.21" @@ -1175,6 +1297,21 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "exr" +version = "1.73.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83197f59927b46c04a183a619b7c29df34e63e63c7869320862268c0ef687e0" +dependencies = [ + "bit_field", + "half", + "lebe", + "miniz_oxide 0.8.8", + "rayon-core", + "smallvec", + "zune-inflate", +] + [[package]] name = "eyre" version = "0.6.12" @@ -1202,6 +1339,15 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "fdeflate" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e6853b52649d4ac5c0bd02320cddc5ba956bdb407c4b75a2c6b75bf51500f8c" +dependencies = [ + "simd-adler32", +] + [[package]] name = "fixedbitset" version = "0.4.2" @@ -1418,6 +1564,16 @@ dependencies = [ "wasi 0.14.2+wasi-0.2.4", ] +[[package]] +name = "gif" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fb2d69b19215e18bb912fa30f7ce15846e301408695e44e0ef719f1da9e19f2" +dependencies = [ + "color_quant", + "weezl", +] + [[package]] name = "gimli" version = "0.28.1" @@ -1449,6 +1605,16 @@ dependencies = [ "tracing", ] +[[package]] +name = "half" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "459196ed295495a68f7d7fe1d84f6c4b7ff0e21fe3017b2f283c6fac3ad803c9" +dependencies = [ + "cfg-if", + "crunchy", +] + [[package]] name = "hashbrown" version = "0.12.3" @@ -1641,7 +1807,7 @@ dependencies = [ "js-sys", "log", "wasm-bindgen", - "windows-core", + "windows-core 0.61.0", ] [[package]] @@ -1771,6 +1937,12 @@ dependencies = [ "syn 2.0.100", ] +[[package]] +name = "icy_sixel" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccc0a9c4770bc47b0a933256a496cfb8b6531f753ea9bccb19c6dff0ff7273fc" + [[package]] name = "ident_case" version = "1.0.1" @@ -1798,6 +1970,45 @@ dependencies = [ "icu_properties", ] +[[package]] +name = "image" +version = "0.25.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db35664ce6b9810857a38a906215e75a9c879f0696556a39f59c62829710251a" +dependencies = [ + "bytemuck", + "byteorder-lite", + "color_quant", + "exr", + "gif", + "image-webp", + "num-traits", + "png", + "qoi", + "ravif", + "rayon", + "rgb", + "tiff", + "zune-core", + "zune-jpeg", +] + +[[package]] +name = "image-webp" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b77d01e822461baa8409e156015a1d91735549f0f2c17691bd2d996bef238f7f" +dependencies = [ + "byteorder-lite", + "quick-error", +] + +[[package]] +name = "imgref" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0263a3d970d5c054ed9312c0057b4f3bde9c0b33836d3637361d4a9e6e7a408" + [[package]] name = "indenter" version = "0.3.3" @@ -1845,6 +2056,17 @@ dependencies = [ "syn 2.0.100", ] +[[package]] +name = "interpolate_name" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c34819042dc3d3971c46c2190835914dfbe0c3c13f61449b2997f4e9722dfa60" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + [[package]] name = "inventory" version = "0.3.20" @@ -1886,6 +2108,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" +dependencies = [ + "either", +] + [[package]] name = "itertools" version = "0.13.0" @@ -1934,6 +2165,22 @@ dependencies = [ "syn 2.0.100", ] +[[package]] +name = "jobserver" +version = "0.1.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38f262f097c174adebe41eb73d66ae9c06b2844fb0da69969647bbddd9b0538a" +dependencies = [ + "getrandom 0.3.2", + "libc", +] + +[[package]] +name = "jpeg-decoder" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5d4a7da358eff58addd2877a45865158f0d78c911d43a5784ceb7bbf52833b0" + [[package]] name = "js-sys" version = "0.3.77" @@ -1992,12 +2239,28 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +[[package]] +name = "lebe" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03087c2bad5e1034e8cace5926dec053fb3790248370865f5117a7d0213354c8" + [[package]] name = "libc" version = "0.2.172" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa" +[[package]] +name = "libfuzzer-sys" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf78f52d400cf2d84a3a973a78a592b4adc535739e0a5597a0da6f0c357adc75" +dependencies = [ + "arbitrary", + "cc", +] + [[package]] name = "libredox" version = "0.1.3" @@ -2071,6 +2334,15 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "loop9" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fae87c125b03c1d2c0150c90365d7d6bcc53fb73a9acaef207d2d065860f062" +dependencies = [ + "imgref", +] + [[package]] name = "lru" version = "0.12.5" @@ -2108,6 +2380,16 @@ dependencies = [ "regex-automata 0.1.10", ] +[[package]] +name = "maybe-rayon" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ea1f30cedd69f0a2954655f7188c6a834246d2bcf1e315e2ac40c4b24dc9519" +dependencies = [ + "cfg-if", + "rayon", +] + [[package]] name = "mcp-types" version = "0.0.0" @@ -2169,6 +2451,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3be647b768db090acb35d5ec5db2b0e1f1de11133ca123b9eacf5137868f892a" dependencies = [ "adler2", + "simd-adler32", ] [[package]] @@ -2257,6 +2540,12 @@ dependencies = [ "nom", ] +[[package]] +name = "noop_proc_macro" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0676bb32a98c1a483ce53e500a81ad9c3d5b3f7c920c28c24e9cb0980d0b5bc8" + [[package]] name = "normalize-line-endings" version = "0.3.0" @@ -2289,6 +2578,17 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" +[[package]] +name = "num-derive" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + [[package]] name = "num-integer" version = "0.1.46" @@ -2298,6 +2598,17 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -2567,6 +2878,19 @@ dependencies = [ "time", ] +[[package]] +name = "png" +version = "0.17.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82151a2fc869e011c153adc57cf2789ccb8d9906ce52c0b39a6b5697749d7526" +dependencies = [ + "bitflags 1.3.2", + "crc32fast", + "fdeflate", + "flate2", + "miniz_oxide 0.8.8", +] + [[package]] name = "portable-atomic" version = "1.11.0" @@ -2661,6 +2985,25 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "profiling" +version = "1.0.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afbdc74edc00b6f6a218ca6a5364d6226a259d4b8ea1af4a0ea063f27e179f4d" +dependencies = [ + "profiling-procmacros", +] + +[[package]] +name = "profiling-procmacros" +version = "1.0.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a65f2e60fbf1063868558d69c6beacf412dc755f9fc020f514b7955fc914fe30" +dependencies = [ + "quote", + "syn 2.0.100", +] + [[package]] name = "pulldown-cmark" version = "0.13.0" @@ -2680,6 +3023,21 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "007d8adb5ddab6f8e3f491ac63566a7d5002cc7ed73901f72057943fa71ae1ae" +[[package]] +name = "qoi" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f6d64c71eb498fe9eae14ce4ec935c555749aef511cca85b5568910d6e48001" +dependencies = [ + "bytemuck", +] + +[[package]] +name = "quick-error" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" + [[package]] name = "quick-xml" version = "0.32.0" @@ -2714,14 +3072,35 @@ dependencies = [ "nibble_vec", ] +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + [[package]] name = "rand" version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97" dependencies = [ - "rand_chacha", - "rand_core", + "rand_chacha 0.9.0", + "rand_core 0.9.3", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", ] [[package]] @@ -2731,7 +3110,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.9.3", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.16", ] [[package]] @@ -2764,6 +3152,92 @@ dependencies = [ "unicode-width 0.2.0", ] +[[package]] +name = "ratatui-image" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3f1d31464920104b247593f008158372d2fdb8165e93a4299cdd6f994448c9a" +dependencies = [ + "base64 0.21.7", + "icy_sixel", + "image", + "rand 0.8.5", + "ratatui", + "rustix 0.38.44", + "thiserror 1.0.69", + "windows", +] + +[[package]] +name = "rav1e" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd87ce80a7665b1cce111f8a16c1f3929f6547ce91ade6addf4ec86a8dda5ce9" +dependencies = [ + "arbitrary", + "arg_enum_proc_macro", + "arrayvec", + "av1-grain", + "bitstream-io", + "built", + "cfg-if", + "interpolate_name", + "itertools 0.12.1", + "libc", + "libfuzzer-sys", + "log", + "maybe-rayon", + "new_debug_unreachable", + "noop_proc_macro", + "num-derive", + "num-traits", + "once_cell", + "paste", + "profiling", + "rand 0.8.5", + "rand_chacha 0.3.1", + "simd_helpers", + "system-deps", + "thiserror 1.0.69", + "v_frame", + "wasm-bindgen", +] + +[[package]] +name = "ravif" +version = "0.11.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6a5f31fcf7500f9401fea858ea4ab5525c99f2322cfcee732c0e6c74208c0c6" +dependencies = [ + "avif-serialize", + "imgref", + "loop9", + "quick-error", + "rav1e", + "rayon", + "rgb", +] + +[[package]] +name = "rayon" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "redox_syscall" version = "0.5.11" @@ -2911,6 +3385,12 @@ dependencies = [ "windows-registry", ] +[[package]] +name = "rgb" +version = "0.8.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57397d16646700483b67d2dd6511d79318f9d057fdbd21a4066aeac8b41d310a" + [[package]] name = "ring" version = "0.17.14" @@ -3348,6 +3828,21 @@ dependencies = [ "libc", ] +[[package]] +name = "simd-adler32" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" + +[[package]] +name = "simd_helpers" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95890f873bec569a0362c235787f3aca6e1e887302ba4840839bcc6459c42da6" +dependencies = [ + "quote", +] + [[package]] name = "simdutf8" version = "0.1.5" @@ -3655,6 +4150,25 @@ dependencies = [ "libc", ] +[[package]] +name = "system-deps" +version = "6.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3e535eb8dded36d55ec13eddacd30dec501792ff23a0b1682c38601b8cf2349" +dependencies = [ + "cfg-expr", + "heck", + "pkg-config", + "toml", + "version-compare", +] + +[[package]] +name = "target-lexicon" +version = "0.12.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1" + [[package]] name = "tempfile" version = "3.19.1" @@ -3754,6 +4268,17 @@ dependencies = [ "once_cell", ] +[[package]] +name = "tiff" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba1310fcea54c6a9a4fd1aad794ecc02c31682f6bfbecdf460bf19533eed1e3e" +dependencies = [ + "flate2", + "jpeg-decoder", + "weezl", +] + [[package]] name = "time" version = "0.3.41" @@ -4178,6 +4703,17 @@ dependencies = [ "serde", ] +[[package]] +name = "v_frame" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f32aaa24bacd11e488aa9ba66369c7cd514885742c9fe08cfe85884db3e92b" +dependencies = [ + "aligned-vec", + "num-traits", + "wasm-bindgen", +] + [[package]] name = "valuable" version = "0.1.1" @@ -4190,6 +4726,12 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" +[[package]] +name = "version-compare" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "852e951cb7832cb45cb1169900d19760cfa39b82bc0ea9c0e5a14ae88411c98b" + [[package]] name = "version_check" version = "0.9.5" @@ -4333,6 +4875,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "weezl" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a751b3277700db47d3e574514de2eced5e54dc8a5436a3bf7a0b248b2cee16f3" + [[package]] name = "wildmatch" version = "2.4.0" @@ -4370,19 +4918,53 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows" +version = "0.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd04d41d93c4992d421894c18c8b43496aa748dd4c081bac0dc93eb0489272b6" +dependencies = [ + "windows-core 0.58.0", + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-core" +version = "0.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ba6d44ec8c2591c134257ce647b7ea6b20335bf6379a27dac5f1641fcf59f99" +dependencies = [ + "windows-implement 0.58.0", + "windows-interface 0.58.0", + "windows-result 0.2.0", + "windows-strings 0.1.0", + "windows-targets 0.52.6", +] + [[package]] name = "windows-core" version = "0.61.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4763c1de310c86d75a878046489e2e5ba02c649d185f21c67d4cf8a56d098980" dependencies = [ - "windows-implement", - "windows-interface", + "windows-implement 0.60.0", + "windows-interface 0.59.1", "windows-link", - "windows-result", + "windows-result 0.3.2", "windows-strings 0.4.0", ] +[[package]] +name = "windows-implement" +version = "0.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bbd5b46c938e506ecbce286b6628a02171d56153ba733b6c741fc627ec9579b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + [[package]] name = "windows-implement" version = "0.60.0" @@ -4394,6 +4976,17 @@ dependencies = [ "syn 2.0.100", ] +[[package]] +name = "windows-interface" +version = "0.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053c4c462dc91d3b1504c6fe5a726dd15e216ba718e84a0e46a88fbe5ded3515" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + [[package]] name = "windows-interface" version = "0.59.1" @@ -4417,11 +5010,20 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4286ad90ddb45071efd1a66dfa43eb02dd0dfbae1545ad6cc3c51cf34d7e8ba3" dependencies = [ - "windows-result", + "windows-result 0.3.2", "windows-strings 0.3.1", "windows-targets 0.53.0", ] +[[package]] +name = "windows-result" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d1043d8214f791817bab27572aaa8af63732e11bf84aa21a45a78d6c317ae0e" +dependencies = [ + "windows-targets 0.52.6", +] + [[package]] name = "windows-result" version = "0.3.2" @@ -4431,6 +5033,16 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-strings" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10" +dependencies = [ + "windows-result 0.2.0", + "windows-targets 0.52.6", +] + [[package]] name = "windows-strings" version = "0.3.1" @@ -4776,3 +5388,27 @@ dependencies = [ "quote", "syn 2.0.100", ] + +[[package]] +name = "zune-core" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f423a2c17029964870cfaabb1f13dfab7d092a62a29a89264f4d36990ca414a" + +[[package]] +name = "zune-inflate" +version = "0.2.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73ab332fe2f6680068f3582b16a24f90ad7096d5d39b974d1c0aff0125116f02" +dependencies = [ + "simd-adler32", +] + +[[package]] +name = "zune-jpeg" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99a5bab8d7dedf81405c4bb1f2b83ea057643d9cb28778cea9eecddeedd2e028" +dependencies = [ + "zune-core", +] diff --git a/codex-rs/core/src/mcp_tool_call.rs b/codex-rs/core/src/mcp_tool_call.rs index 4da5b2b77c..61a51a0e7a 100644 --- a/codex-rs/core/src/mcp_tool_call.rs +++ b/codex-rs/core/src/mcp_tool_call.rs @@ -50,51 +50,18 @@ pub(crate) async fn handle_mcp_tool_call( notify_mcp_tool_call_event(sess, sub_id, tool_call_begin_event).await; // Perform the tool call. - let (tool_call_end_event, tool_call_err) = match sess + let result = sess .call_tool(&server, &tool_name, arguments_value, timeout) .await - { - Ok(result) => ( - EventMsg::McpToolCallEnd(McpToolCallEndEvent { - call_id, - success: !result.is_error.unwrap_or(false), - result: Some(result), - }), - None, - ), - Err(e) => ( - EventMsg::McpToolCallEnd(McpToolCallEndEvent { - call_id, - success: false, - result: None, - }), - Some(e), - ), - }; + .map_err(|e| format!("tool call error: {e}")); + let tool_call_end_event = EventMsg::McpToolCallEnd(McpToolCallEndEvent { + call_id: call_id.clone(), + result: result.clone(), + }); notify_mcp_tool_call_event(sess, sub_id, tool_call_end_event.clone()).await; - let EventMsg::McpToolCallEnd(McpToolCallEndEvent { - call_id, - success, - result, - }) = tool_call_end_event - else { - unimplemented!("unexpected event type"); - }; - ResponseInputItem::FunctionCallOutput { - call_id, - output: FunctionCallOutputPayload { - content: result.map_or_else( - || format!("err: {tool_call_err:?}"), - |result| { - serde_json::to_string(&result) - .unwrap_or_else(|e| format!("JSON serialization error: {e}")) - }, - ), - success: Some(success), - }, - } + ResponseInputItem::McpToolCallOutput { call_id, result } } async fn notify_mcp_tool_call_event(sess: &Session, sub_id: &str, event: EventMsg) { diff --git a/codex-rs/core/src/models.rs b/codex-rs/core/src/models.rs index ab213fd529..ccc550e8e5 100644 --- a/codex-rs/core/src/models.rs +++ b/codex-rs/core/src/models.rs @@ -1,6 +1,7 @@ use std::collections::HashMap; use base64::Engine; +use mcp_types::CallToolResult; use serde::Deserialize; use serde::Serialize; use serde::ser::Serializer; @@ -18,6 +19,10 @@ pub enum ResponseInputItem { call_id: String, output: FunctionCallOutputPayload, }, + McpToolCallOutput { + call_id: String, + result: Result, + }, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -77,6 +82,19 @@ impl From for ResponseItem { ResponseInputItem::FunctionCallOutput { call_id, output } => { Self::FunctionCallOutput { call_id, output } } + ResponseInputItem::McpToolCallOutput { call_id, result } => Self::FunctionCallOutput { + call_id, + output: FunctionCallOutputPayload { + success: Some(result.is_ok()), + content: result.map_or_else( + |tool_call_err| format!("err: {tool_call_err:?}"), + |result| { + serde_json::to_string(&result) + .unwrap_or_else(|e| format!("JSON serialization error: {e}")) + }, + ), + }, + }, } } } diff --git a/codex-rs/core/src/protocol.rs b/codex-rs/core/src/protocol.rs index 2a922cba6c..1b9871edd8 100644 --- a/codex-rs/core/src/protocol.rs +++ b/codex-rs/core/src/protocol.rs @@ -396,10 +396,17 @@ pub struct McpToolCallBeginEvent { pub struct McpToolCallEndEvent { /// Identifier for the corresponding McpToolCallBegin that finished. pub call_id: String, - /// Whether the tool call was successful. If `false`, `result` might not be present. - pub success: bool, /// Result of the tool call. Note this could be an error. - pub result: Option, + pub result: Result, +} + +impl McpToolCallEndEvent { + pub fn is_success(&self) -> bool { + match &self.result { + Ok(result) => !result.is_error.unwrap_or(false), + Err(_) => false, + } + } } #[derive(Debug, Clone, Deserialize, Serialize)] diff --git a/codex-rs/exec/src/event_processor.rs b/codex-rs/exec/src/event_processor.rs index 676b47d64f..352275bf43 100644 --- a/codex-rs/exec/src/event_processor.rs +++ b/codex-rs/exec/src/event_processor.rs @@ -242,11 +242,9 @@ impl EventProcessor { invocation.style(self.bold), ); } - EventMsg::McpToolCallEnd(McpToolCallEndEvent { - call_id, - success, - result, - }) => { + EventMsg::McpToolCallEnd(tool_call_end_event) => { + let is_success = tool_call_end_event.is_success(); + let McpToolCallEndEvent { call_id, result } = tool_call_end_event; // Retrieve start time and invocation for duration calculation and labeling. let info = self.call_id_to_tool_call.remove(&call_id); @@ -261,13 +259,13 @@ impl EventProcessor { (String::new(), format!("tool('{call_id}')")) }; - let status_str = if success { "success" } else { "failed" }; - let title_style = if success { self.green } else { self.red }; + let status_str = if is_success { "success" } else { "failed" }; + let title_style = if is_success { self.green } else { self.red }; let title = format!("{invocation} {status_str}{duration}:"); ts_println!("{}", title.style(title_style)); - if let Some(res) = result { + if let Ok(res) = result { let val: serde_json::Value = res.into(); let pretty = serde_json::to_string_pretty(&val).unwrap_or_else(|_| val.to_string()); diff --git a/codex-rs/tui/Cargo.toml b/codex-rs/tui/Cargo.toml index c7a8361faf..5886ce69dc 100644 --- a/codex-rs/tui/Cargo.toml +++ b/codex-rs/tui/Cargo.toml @@ -16,6 +16,7 @@ workspace = true [dependencies] anyhow = "1" +base64 = "0.22.1" clap = { version = "4", features = ["derive"] } codex-ansi-escape = { path = "../ansi-escape" } codex-core = { path = "../core" } @@ -23,6 +24,7 @@ codex-common = { path = "../common", features = ["cli", "elapsed"] } codex-linux-sandbox = { path = "../linux-sandbox" } color-eyre = "0.6.3" crossterm = { version = "0.28.1", features = ["bracketed-paste"] } +image = { version = "^0.25.6", default-features = false, features = ["jpeg"] } lazy_static = "1" mcp-types = { path = "../mcp-types" } path-clean = "1.0.1" @@ -30,6 +32,7 @@ ratatui = { version = "0.29.0", features = [ "unstable-widget-ref", "unstable-rendered-line-info", ] } +ratatui-image = "8.0.0" regex = "1" serde_json = "1" shlex = "1.3.0" diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index 189f399447..4819be3809 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -343,11 +343,9 @@ impl ChatWidget<'_> { .add_active_mcp_tool_call(call_id, server, tool, arguments); self.request_redraw(); } - EventMsg::McpToolCallEnd(McpToolCallEndEvent { - call_id, - success, - result, - }) => { + EventMsg::McpToolCallEnd(mcp_tool_call_end_event) => { + let success = mcp_tool_call_end_event.is_success(); + let McpToolCallEndEvent { call_id, result } = mcp_tool_call_end_event; self.conversation_history .record_completed_mcp_tool_call(call_id, success, result); self.request_redraw(); diff --git a/codex-rs/tui/src/conversation_history_widget.rs b/codex-rs/tui/src/conversation_history_widget.rs index d69f4db88e..9242e00389 100644 --- a/codex-rs/tui/src/conversation_history_widget.rs +++ b/codex-rs/tui/src/conversation_history_widget.rs @@ -293,15 +293,8 @@ impl ConversationHistoryWidget { &mut self, call_id: String, success: bool, - result: Option, + result: Result, ) { - // Convert result into serde_json::Value early so we don't have to - // worry about lifetimes inside the match arm. - let result_val = result.map(|r| { - serde_json::to_value(r) - .unwrap_or_else(|_| serde_json::Value::String("".into())) - }); - let width = self.cached_width.get(); for entry in self.entries.iter_mut() { if let HistoryCell::ActiveMcpToolCall { @@ -318,7 +311,7 @@ impl ConversationHistoryWidget { invocation.clone(), *start, success, - result_val, + result, ); entry.cell = completed; diff --git a/codex-rs/tui/src/history_cell.rs b/codex-rs/tui/src/history_cell.rs index c2938f4b85..41c2049313 100644 --- a/codex-rs/tui/src/history_cell.rs +++ b/codex-rs/tui/src/history_cell.rs @@ -1,24 +1,32 @@ +use crate::cell_widget::CellWidget; +use crate::exec_command::escape_command; +use crate::markdown::append_markdown; +use crate::text_block::TextBlock; +use base64::Engine; use codex_ansi_escape::ansi_escape_line; use codex_common::elapsed::format_duration; use codex_core::config::Config; use codex_core::protocol::FileChange; use codex_core::protocol::SessionConfiguredEvent; +use image::DynamicImage; +use image::GenericImageView; +use image::ImageReader; +use lazy_static::lazy_static; use ratatui::prelude::*; use ratatui::style::Color; use ratatui::style::Modifier; use ratatui::style::Style; use ratatui::text::Line as RtLine; use ratatui::text::Span as RtSpan; - -use crate::cell_widget::CellWidget; -use crate::text_block::TextBlock; +use ratatui_image::Image as TuiImage; +use ratatui_image::Resize as ImgResize; +use ratatui_image::picker::ProtocolType; use std::collections::HashMap; +use std::io::Cursor; use std::path::PathBuf; use std::time::Duration; use std::time::Instant; - -use crate::exec_command::escape_command; -use crate::markdown::append_markdown; +use tracing::error; pub(crate) struct CommandOutput { pub(crate) exit_code: i32, @@ -73,8 +81,24 @@ pub(crate) enum HistoryCell { view: TextBlock, }, - /// Completed MCP tool call. - CompletedMcpToolCall { view: TextBlock }, + /// Completed MCP tool call where we show the result serialized as JSON. + CompletedMcpToolCallWithTextOutput { view: TextBlock }, + + /// Completed MCP tool call where the result is an image. + /// Admittedly, [mcp_types::CallToolResult] can have multiple content types, + /// which could be a mix of text and images, so we need to tighten this up. + // NOTE: For image output we keep the *original* image around and lazily + // compute a resized copy that fits the available cell width. Caching the + // resized version avoids doing the potentially expensive rescale twice + // because the scroll-view first calls `height()` for layouting and then + // `render_window()` for painting. + CompletedMcpToolCallWithImageOutput { + image: DynamicImage, + /// Cached data derived from the current terminal width. The cache is + /// invalidated whenever the width changes (e.g. when the user + /// resizes the window). + render_cache: std::cell::RefCell>, + }, /// Background event. BackgroundEvent { view: TextBlock }, @@ -284,13 +308,61 @@ impl HistoryCell { } } + fn try_new_completed_mcp_tool_call_with_image_output( + result: &Result, + ) -> Option { + match result { + Ok(mcp_types::CallToolResult { content, .. }) => { + if let Some(mcp_types::CallToolResultContent::ImageContent(image)) = content.first() + { + let raw_data = + match base64::engine::general_purpose::STANDARD.decode(&image.data) { + Ok(data) => data, + Err(e) => { + error!("Failed to decode image data: {e}"); + return None; + } + }; + let reader = match ImageReader::new(Cursor::new(raw_data)).with_guessed_format() + { + Ok(reader) => reader, + Err(e) => { + error!("Failed to guess image format: {e}"); + return None; + } + }; + + let image = match reader.decode() { + Ok(image) => image, + Err(e) => { + error!("Image decoding failed: {e}"); + return None; + } + }; + + Some(HistoryCell::CompletedMcpToolCallWithImageOutput { + image, + render_cache: std::cell::RefCell::new(None), + }) + } else { + None + } + } + _ => None, + } + } + pub(crate) fn new_completed_mcp_tool_call( fq_tool_name: String, invocation: String, start: Instant, success: bool, - result: Option, + result: Result, ) -> Self { + if let Some(cell) = Self::try_new_completed_mcp_tool_call_with_image_output(&result) { + return cell; + } + let duration = format_duration(start.elapsed()); let status_str = if success { "success" } else { "failed" }; let title_line = Line::from(vec![ @@ -302,7 +374,14 @@ impl HistoryCell { lines.push(title_line); lines.push(Line::from(format!("$ {invocation}"))); - if let Some(res_val) = result { + // Convert result into serde_json::Value early so we don't have to + // worry about lifetimes inside the match arm. + let result_val = result.map(|r| { + serde_json::to_value(r) + .unwrap_or_else(|_| serde_json::Value::String("".into())) + }); + + if let Ok(res_val) = result_val { let json_pretty = serde_json::to_string_pretty(&res_val).unwrap_or_else(|_| res_val.to_string()); let mut iter = json_pretty.lines(); @@ -317,7 +396,7 @@ impl HistoryCell { lines.push(Line::from("")); - HistoryCell::CompletedMcpToolCall { + HistoryCell::CompletedMcpToolCallWithTextOutput { view: TextBlock::new(lines), } } @@ -424,10 +503,14 @@ impl CellWidget for HistoryCell { | HistoryCell::ErrorEvent { view } | HistoryCell::SessionInfo { view } | HistoryCell::CompletedExecCommand { view } - | HistoryCell::CompletedMcpToolCall { view } + | HistoryCell::CompletedMcpToolCallWithTextOutput { view } | HistoryCell::PendingPatch { view } | HistoryCell::ActiveExecCommand { view, .. } | HistoryCell::ActiveMcpToolCall { view, .. } => view.height(width), + HistoryCell::CompletedMcpToolCallWithImageOutput { + image, + render_cache, + } => ensure_image_cache(image, width, render_cache), } } @@ -441,12 +524,41 @@ impl CellWidget for HistoryCell { | HistoryCell::ErrorEvent { view } | HistoryCell::SessionInfo { view } | HistoryCell::CompletedExecCommand { view } - | HistoryCell::CompletedMcpToolCall { view } + | HistoryCell::CompletedMcpToolCallWithTextOutput { view } | HistoryCell::PendingPatch { view } | HistoryCell::ActiveExecCommand { view, .. } | HistoryCell::ActiveMcpToolCall { view, .. } => { view.render_window(first_visible_line, area, buf) } + HistoryCell::CompletedMcpToolCallWithImageOutput { + image, + render_cache, + } => { + // Ensure we have a cached, resized copy that matches the current width. + // `height()` should have prepared the cache, but if something invalidated it + // (e.g. the first `render_window()` call happens *before* `height()` after a + // resize) we rebuild it here. + + let width_cells = area.width; + + // Ensure the cache is up-to-date and extract the scaled image. + let _ = ensure_image_cache(image, width_cells, render_cache); + + let Some(resized) = render_cache + .borrow() + .as_ref() + .map(|c| c.scaled_image.clone()) + else { + return; + }; + + let picker = &*TERMINAL_PICKER; + + if let Ok(protocol) = picker.new_protocol(resized, area, ImgResize::Fit(None)) { + let img_widget = TuiImage::new(&protocol); + img_widget.render(area, buf); + } + } } } } @@ -482,3 +594,120 @@ fn create_diff_summary(changes: HashMap) -> Vec { summaries } + +// ------------------------------------- +// Helper types for image rendering +// ------------------------------------- + +/// Cached information for rendering an image inside a conversation cell. +/// +/// The cache ties the resized image to a *specific* content width (in +/// terminal cells). Whenever the terminal is resized and the width changes +/// we need to re-compute the scaled variant so that it still fits the +/// available space. Keeping the resized copy around saves a costly rescale +/// between the back-to-back `height()` and `render_window()` calls that the +/// scroll-view performs while laying out the UI. +pub(crate) struct ImageRenderCache { + /// Width in *terminal cells* the cached image was generated for. + width_cells: u16, + /// Height in *terminal rows* that the conversation cell must occupy so + /// the whole image becomes visible. + height_rows: usize, + /// The resized image that fits the given width / height constraints. + scaled_image: DynamicImage, +} + +lazy_static! { + static ref TERMINAL_PICKER: ratatui_image::picker::Picker = { + use ratatui_image::picker::Picker; + use ratatui_image::picker::cap_parser::QueryStdioOptions; + + // Ask the terminal for capabilities and explicit font size. Request the + // Kitty *text-sizing protocol* as a fallback mechanism for terminals + // (like iTerm2) that do not reply to the standard CSI 16/18 queries. + match Picker::from_query_stdio_with_options(QueryStdioOptions { + text_sizing_protocol: true, + }) { + Ok(picker) => picker, + Err(err) => { + // Fall back to the conservative default that assumes ~8×16 px cells. + // Still better than breaking the build in a headless test run. + tracing::warn!("terminal capability query failed: {err:?}; using default font size"); + Picker::from_fontsize((8, 16)) + } + } + }; +} + +/// Resize `image` to fit into `width_cells`×10-rows keeping the original aspect +/// ratio. The function updates `render_cache` and returns the number of rows +/// (<= 10) the picture will occupy. +fn ensure_image_cache( + image: &DynamicImage, + width_cells: u16, + render_cache: &std::cell::RefCell>, +) -> usize { + if let Some(cache) = render_cache.borrow().as_ref() { + if cache.width_cells == width_cells { + return cache.height_rows; + } + } + + let picker = &*TERMINAL_PICKER; + let (char_w_px, char_h_px) = picker.font_size(); + + // Heuristic to compensate for Hi-DPI terminals (iTerm2 on Retina Mac) that + // report logical pixels (≈ 8×16) while the iTerm2 graphics protocol + // expects *device* pixels. Empirically the device-pixel-ratio is almost + // always 2 on macOS Retina panels. + let hidpi_scale = if picker.protocol_type() == ProtocolType::Iterm2 { + 2.0f64 + } else { + 1.0 + }; + + // The fallback Halfblocks protocol encodes two pixel rows per cell, so each + // terminal *row* represents only half the (possibly scaled) font height. + let effective_char_h_px: f64 = if picker.protocol_type() == ProtocolType::Halfblocks { + (char_h_px as f64) * hidpi_scale / 2.0 + } else { + (char_h_px as f64) * hidpi_scale + }; + + let char_w_px_f64 = (char_w_px as f64) * hidpi_scale; + + const MAX_ROWS: f64 = 10.0; + let max_height_px: f64 = effective_char_h_px * MAX_ROWS; + + let (orig_w_px, orig_h_px) = { + let (w, h) = image.dimensions(); + (w as f64, h as f64) + }; + + if orig_w_px == 0.0 || orig_h_px == 0.0 || width_cells == 0 { + *render_cache.borrow_mut() = None; + return 0; + } + + let max_w_px = char_w_px_f64 * width_cells as f64; + let scale_w = max_w_px / orig_w_px; + let scale_h = max_height_px / orig_h_px; + let scale = scale_w.min(scale_h).min(1.0); + + use image::imageops::FilterType; + let scaled_w_px = (orig_w_px * scale).round().max(1.0) as u32; + let scaled_h_px = (orig_h_px * scale).round().max(1.0) as u32; + + let scaled_image = image.resize(scaled_w_px, scaled_h_px, FilterType::Lanczos3); + + let height_rows = ((scaled_h_px as f64 / effective_char_h_px).ceil()) as usize; + + let new_cache = ImageRenderCache { + width_cells, + height_rows, + scaled_image, + }; + *render_cache.borrow_mut() = Some(new_cache); + + height_rows +} From a32d305ae6a8c8680932b5259f7dd7e5a086db26 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Thu, 29 May 2025 14:57:55 -0700 Subject: [PATCH 06/41] fix: update UI treatment of slash command menu to match that of the TS CLI (#1161) Uses the same colors as in the TypeScript CLI: ![image](https://github.com/user-attachments/assets/919cd472-ffb4-4654-a46a-d84f0cd9c097) Now it is also readable on a light theme, e.g., in Ghostty: ![image](https://github.com/user-attachments/assets/468c37b0-ea63-4455-9b48-73dc2c95f0f6) --- codex-rs/tui/src/bottom_pane/command_popup.rs | 29 ++++++++++--------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/codex-rs/tui/src/bottom_pane/command_popup.rs b/codex-rs/tui/src/bottom_pane/command_popup.rs index 505a4bc699..0dcb98865c 100644 --- a/codex-rs/tui/src/bottom_pane/command_popup.rs +++ b/codex-rs/tui/src/bottom_pane/command_popup.rs @@ -4,6 +4,7 @@ use ratatui::buffer::Buffer; use ratatui::layout::Rect; use ratatui::style::Color; use ratatui::style::Style; +use ratatui::style::Stylize; use ratatui::widgets::Block; use ratatui::widgets::BorderType; use ratatui::widgets::Borders; @@ -147,8 +148,6 @@ impl CommandPopup { impl WidgetRef for CommandPopup { fn render_ref(&self, area: Rect, buf: &mut Buffer) { - let style = Style::default().bg(Color::Blue).fg(Color::White); - let matches = self.filtered_commands(); let mut rows: Vec = Vec::new(); @@ -157,21 +156,25 @@ impl WidgetRef for CommandPopup { if visible_matches.is_empty() { rows.push(Row::new(vec![ - Cell::from("").style(style), - Cell::from("No matching commands").style(style.add_modifier(Modifier::ITALIC)), + Cell::from(""), + Cell::from("No matching commands").add_modifier(Modifier::ITALIC), ])); } else { + let default_style = Style::default(); + let command_style = Style::default().fg(Color::LightBlue); for (idx, cmd) in visible_matches.iter().enumerate() { - let highlight = Style::default().bg(Color::White).fg(Color::Blue); - let cmd_style = if Some(idx) == self.selected_idx { - highlight + let (cmd_style, desc_style) = if Some(idx) == self.selected_idx { + ( + command_style.bg(Color::DarkGray), + default_style.bg(Color::DarkGray), + ) } else { - style + (command_style, default_style) }; rows.push(Row::new(vec![ - Cell::from(cmd.command().to_string()).style(cmd_style), - Cell::from(cmd.description().to_string()).style(style), + Cell::from(format!("/{}", cmd.command())).style(cmd_style), + Cell::from(cmd.description().to_string()).style(desc_style), ])); } } @@ -182,13 +185,11 @@ impl WidgetRef for CommandPopup { rows, [Constraint::Length(FIRST_COLUMN_WIDTH), Constraint::Min(10)], ) - .style(style) - .column_spacing(1) + .column_spacing(0) .block( Block::default() .borders(Borders::ALL) - .border_type(BorderType::Rounded) - .style(style), + .border_type(BorderType::Rounded), ); table.render(area, buf); From 8c1902b56204e84b463240e49aa920a480ef457f Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Thu, 29 May 2025 15:34:06 -0700 Subject: [PATCH 07/41] chore: update GitHub workflow for native artifacts for npm release (#1162) Among other things, this picks up this UI treatment fix: https://github.com/openai/codex/pull/1161 --- codex-cli/scripts/install_native_deps.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codex-cli/scripts/install_native_deps.sh b/codex-cli/scripts/install_native_deps.sh index 09c1553228..c1697fb5fe 100755 --- a/codex-cli/scripts/install_native_deps.sh +++ b/codex-cli/scripts/install_native_deps.sh @@ -65,7 +65,7 @@ mkdir -p "$BIN_DIR" # Until we start publishing stable GitHub releases, we have to grab the binaries # from the GitHub Action that created them. Update the URL below to point to the # appropriate workflow run: -WORKFLOW_URL="https://github.com/openai/codex/actions/runs/15280451034" +WORKFLOW_URL="https://github.com/openai/codex/actions/runs/15334411824" WORKFLOW_ID="${WORKFLOW_URL##*/}" ARTIFACTS_DIR="$(mktemp -d)" From 92957c47fb379422f60ca991c02439bca6b58343 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Thu, 29 May 2025 15:35:14 -0700 Subject: [PATCH 08/41] fix: update justfile to facilitate running CLIs from source and formatting source code (#1163) --- codex-rs/justfile | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/codex-rs/justfile b/codex-rs/justfile index 61339a2320..12088585ff 100644 --- a/codex-rs/justfile +++ b/codex-rs/justfile @@ -2,14 +2,18 @@ help: just -l -# Install the `codex-tui` binary -install: - cargo install --path tui +# `codex` +codex *args: + cargo run --bin codex -- {{args}} -# Run the TUI app +# `codex exec` +exec *args: + cargo run --bin codex -- exec {{args}} + +# `codex tui` tui *args: cargo run --bin codex -- tui {{args}} -# Run the Proto app -proto *args: - cargo run --bin codex -- proto {{args}} +# format code +fmt: + cargo fmt -- --config imports_granularity=Item From 828e2062c2cbd7a4ec9414facba5a2136a1f8ff9 Mon Sep 17 00:00:00 2001 From: Fouad Matin <169186268+fouad-openai@users.noreply.github.com> Date: Thu, 29 May 2025 16:55:19 -0700 Subject: [PATCH 09/41] fix(codex-rs): use codex-mini-latest as default (#1164) --- codex-rs/README.md | 4 ++-- codex-rs/common/src/config_override.rs | 4 ++-- codex-rs/core/src/flags.rs | 2 +- codex-rs/core/src/protocol.rs | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/codex-rs/README.md b/codex-rs/README.md index a0e3f5846e..a26f5b6d1d 100644 --- a/codex-rs/README.md +++ b/codex-rs/README.md @@ -32,7 +32,7 @@ The `config.toml` file supports the following options: The model that Codex should use. ```toml -model = "o3" # overrides the default of "o4-mini" +model = "o3" # overrides the default of "codex-mini-latest" ``` ### model_provider @@ -155,7 +155,7 @@ Users can specify config values at multiple levels. Order of precedence is as fo 1. custom command-line argument, e.g., `--model o3` 2. as part of a profile, where the `--profile` is specified via a CLI (or in the config file itself) 3. as an entry in `config.toml`, e.g., `model = "o3"` -4. the default value that comes with Codex CLI (i.e., Codex CLI defaults to `o4-mini`) +4. the default value that comes with Codex CLI (i.e., Codex CLI defaults to `codex-mini-latest`) ### sandbox_permissions diff --git a/codex-rs/common/src/config_override.rs b/codex-rs/common/src/config_override.rs index bd2c036940..610195d6d1 100644 --- a/codex-rs/common/src/config_override.rs +++ b/codex-rs/common/src/config_override.rs @@ -23,7 +23,7 @@ pub struct CliConfigOverrides { /// parse as JSON, the raw string is used as a literal. /// /// Examples: - /// - `-c model="o4-mini"` + /// - `-c model="o3"` /// - `-c 'sandbox_permissions=["disk-full-read-access"]'` /// - `-c shell_environment_policy.inherit=all` #[arg( @@ -61,7 +61,7 @@ impl CliConfigOverrides { // Attempt to parse as JSON. If that fails, treat it as a raw // string. This allows convenient usage such as - // `-c model=o4-mini` without the quotes. + // `-c model=o3` without the quotes. let value: Value = match parse_toml_value(value_str) { Ok(v) => v, Err(_) => Value::String(value_str.to_string()), diff --git a/codex-rs/core/src/flags.rs b/codex-rs/core/src/flags.rs index e8cc973c99..c21ef67026 100644 --- a/codex-rs/core/src/flags.rs +++ b/codex-rs/core/src/flags.rs @@ -3,7 +3,7 @@ use std::time::Duration; use env_flags::env_flags; env_flags! { - pub OPENAI_DEFAULT_MODEL: &str = "o4-mini"; + pub OPENAI_DEFAULT_MODEL: &str = "codex-mini-latest"; pub OPENAI_API_BASE: &str = "https://api.openai.com/v1"; /// Fallback when the provider-specific key is not set. diff --git a/codex-rs/core/src/protocol.rs b/codex-rs/core/src/protocol.rs index 1b9871edd8..fc18f1d821 100644 --- a/codex-rs/core/src/protocol.rs +++ b/codex-rs/core/src/protocol.rs @@ -561,7 +561,7 @@ mod tests { id: "1234".to_string(), msg: EventMsg::SessionConfigured(SessionConfiguredEvent { session_id, - model: "o4-mini".to_string(), + model: "codex-mini-latest".to_string(), history_log_id: 0, history_entry_count: 0, }), @@ -569,7 +569,7 @@ mod tests { let serialized = serde_json::to_string(&event).unwrap(); assert_eq!( serialized, - r#"{"id":"1234","msg":{"type":"session_configured","session_id":"67e55044-10b1-426f-9247-bb680e5fe0c8","model":"o4-mini","history_log_id":0,"history_entry_count":0}}"# + r#"{"id":"1234","msg":{"type":"session_configured","session_id":"67e55044-10b1-426f-9247-bb680e5fe0c8","model":"codex-mini-latest","history_log_id":0,"history_entry_count":0}}"# ); } } From bdfa95ed313c88c219242e66182a9d044054a177 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Thu, 29 May 2025 16:59:35 -0700 Subject: [PATCH 10/41] docs: split the config-related portion of codex-rs/README.md into its own config.md file (#1165) Also updated the overview on `codex-rs/README.md` while here. --- codex-rs/README.md | 397 +++------------------------------------------ codex-rs/config.md | 377 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 398 insertions(+), 376 deletions(-) create mode 100644 codex-rs/config.md diff --git a/codex-rs/README.md b/codex-rs/README.md index a26f5b6d1d..7126d60b4f 100644 --- a/codex-rs/README.md +++ b/codex-rs/README.md @@ -1,392 +1,37 @@ -# codex-rs +# Codex CLI (Rust Implementation) -April 24, 2025 +We provide Codex CLI as a standalone, native executable to ensure a zero-dependency install. -Today, Codex CLI is written in TypeScript and requires Node.js 22+ to run it. For a number of users, this runtime requirement inhibits adoption: they would be better served by a standalone executable. As maintainers, we want Codex to run efficiently in a wide range of environments with minimal overhead. We also want to take advantage of operating system-specific APIs to provide better sandboxing, where possible. +## Installing Codex -To that end, we are moving forward with a Rust implementation of Codex CLI contained in this folder, which has the following benefits: +Today, the easiest way to install Codex is via `npm`, though we plan to publish Codex to other package managers soon. -- The CLI compiles to small, standalone, platform-specific binaries. -- Can make direct, native calls to [seccomp](https://man7.org/linux/man-pages/man2/seccomp.2.html) and [landlock](https://man7.org/linux/man-pages/man7/landlock.7.html) in order to support sandboxing on Linux. -- No runtime garbage collection, resulting in lower memory consumption and better, more predictable performance. - -Currently, the Rust implementation is materially behind the TypeScript implementation in functionality, so continue to use the TypeScript implementation for the time being. We will publish native executables via GitHub Releases as soon as we feel the Rust version is usable. - -## Code Organization - -This folder is the root of a Cargo workspace. It contains quite a bit of experimental code, but here are the key crates: - -- [`core/`](./core) contains the business logic for Codex. Ultimately, we hope this to be a library crate that is generally useful for building other Rust/native applications that use Codex. -- [`exec/`](./exec) "headless" CLI for use in automation. -- [`tui/`](./tui) CLI that launches a fullscreen TUI built with [Ratatui](https://ratatui.rs/). -- [`cli/`](./cli) CLI multitool that provides the aforementioned CLIs via subcommands. - -## Config - -The CLI can be configured via a file named `config.toml`. By default, configuration is read from `~/.codex/config.toml`, though the `CODEX_HOME` environment variable can be used to specify a directory other than `~/.codex`. - -The `config.toml` file supports the following options: - -### model - -The model that Codex should use. - -```toml -model = "o3" # overrides the default of "codex-mini-latest" -``` - -### model_provider - -Codex comes bundled with a number of "model providers" predefined. This config value is a string that indicates which provider to use. You can also define your own providers via `model_providers`. - -For example, if you are running ollama with Mistral locally, then you would need to add the following to your config: - -```toml -model = "mistral" -model_provider = "ollama" +```shell +npm i -g @openai/codex@native +codex ``` -because the following definition for `ollama` is included in Codex: - -```toml -[model_providers.ollama] -name = "Ollama" -base_url = "http://localhost:11434/v1" -wire_api = "chat" -``` - -This option defaults to `"openai"` and the corresponding provider is defined as follows: - -```toml -[model_providers.openai] -name = "OpenAI" -base_url = "https://api.openai.com/v1" -env_key = "OPENAI_API_KEY" -wire_api = "responses" -``` - -### model_providers - -This option lets you override and amend the default set of model providers bundled with Codex. This value is a map where the key is the value to use with `model_provider` to select the correspodning provider. - -For example, if you wanted to add a provider that uses the OpenAI 4o model via the chat completions API, then you +You can also download a platform-specific release directly from our [GitHub Releases](https://github.com/openai/codex/releases). -```toml -# Recall that in TOML, root keys must be listed before tables. -model = "gpt-4o" -model_provider = "openai-chat-completions" - -[model_providers.openai-chat-completions] -# Name of the provider that will be displayed in the Codex UI. -name = "OpenAI using Chat Completions" -# The path `/chat/completions` will be amended to this URL to make the POST -# request for the chat completions. -base_url = "https://api.openai.com/v1" -# If `env_key` is set, identifies an environment variable that must be set when -# using Codex with this provider. The value of the environment variable must be -# non-empty and will be used in the `Bearer TOKEN` HTTP header for the POST request. -env_key = "OPENAI_API_KEY" -# valid values for wire_api are "chat" and "responses". -wire_api = "chat" -``` - -### approval_policy - -Determines when the user should be prompted to approve whether Codex can execute a command: - -```toml -# This is analogous to --suggest in the TypeScript Codex CLI -approval_policy = "unless-allow-listed" -``` - -```toml -# If the command fails when run in the sandbox, Codex asks for permission to -# retry the command outside the sandbox. -approval_policy = "on-failure" -``` - -```toml -# User is never prompted: if the command fails, Codex will automatically try -# something out. Note the `exec` subcommand always uses this mode. -approval_policy = "never" -``` - -### profiles - -A _profile_ is a collection of configuration values that can be set together. Multiple profiles can be defined in `config.toml` and you can specify the one you -want to use at runtime via the `--profile` flag. - -Here is an example of a `config.toml` that defines multiple profiles: - -```toml -model = "o3" -approval_policy = "unless-allow-listed" -sandbox_permissions = ["disk-full-read-access"] -disable_response_storage = false - -# Setting `profile` is equivalent to specifying `--profile o3` on the command -# line, though the `--profile` flag can still be used to override this value. -profile = "o3" - -[model_providers.openai-chat-completions] -name = "OpenAI using Chat Completions" -base_url = "https://api.openai.com/v1" -env_key = "OPENAI_API_KEY" -wire_api = "chat" - -[profiles.o3] -model = "o3" -model_provider = "openai" -approval_policy = "never" - -[profiles.gpt3] -model = "gpt-3.5-turbo" -model_provider = "openai-chat-completions" - -[profiles.zdr] -model = "o3" -model_provider = "openai" -approval_policy = "on-failure" -disable_response_storage = true -``` - -Users can specify config values at multiple levels. Order of precedence is as follows: - -1. custom command-line argument, e.g., `--model o3` -2. as part of a profile, where the `--profile` is specified via a CLI (or in the config file itself) -3. as an entry in `config.toml`, e.g., `model = "o3"` -4. the default value that comes with Codex CLI (i.e., Codex CLI defaults to `codex-mini-latest`) - -### sandbox_permissions - -List of permissions to grant to the sandbox that Codex uses to execute untrusted commands: - -```toml -# This is comparable to --full-auto in the TypeScript Codex CLI, though -# specifying `disk-write-platform-global-temp-folder` adds /tmp as a writable -# folder in addition to $TMPDIR. -sandbox_permissions = [ - "disk-full-read-access", - "disk-write-platform-user-temp-folder", - "disk-write-platform-global-temp-folder", - "disk-write-cwd", -] -``` - -To add additional writable folders, use `disk-write-folder`, which takes a parameter (this can be specified multiple times): - -```toml -sandbox_permissions = [ - # ... - "disk-write-folder=/Users/mbolin/.pyenv/shims", -] -``` - -### mcp_servers - -Defines the list of MCP servers that Codex can consult for tool use. Currently, only servers that are launched by executing a program that communicate over stdio are supported. For servers that use the SSE transport, consider an adapter like [mcp-proxy](https://github.com/sparfenyuk/mcp-proxy). - -**Note:** Codex may cache the list of tools and resources from an MCP server so that Codex can include this information in context at startup without spawning all the servers. This is designed to save resources by loading MCP servers lazily. - -This config option is comparable to how Claude and Cursor define `mcpServers` in their respective JSON config files, though because Codex uses TOML for its config language, the format is slightly different. For example, the following config in JSON: - -```json -{ - "mcpServers": { - "server-name": { - "command": "npx", - "args": ["-y", "mcp-server"], - "env": { - "API_KEY": "value" - } - } - } -} -``` - -Should be represented as follows in `~/.codex/config.toml`: - -```toml -# IMPORTANT: the top-level key is `mcp_servers` rather than `mcpServers`. -[mcp_servers.server-name] -command = "npx" -args = ["-y", "mcp-server"] -env = { "API_KEY" = "value" } -``` - -### disable_response_storage - -Currently, customers whose accounts are set to use Zero Data Retention (ZDR) must set `disable_response_storage` to `true` so that Codex uses an alternative to the Responses API that works with ZDR: - -```toml -disable_response_storage = true -``` - -### shell_environment_policy - -Codex spawns subprocesses (e.g. when executing a `local_shell` tool-call suggested by the assistant). By default it passes **only a minimal core subset** of your environment to those subprocesses to avoid leaking credentials. You can tune this behavior via the **`shell_environment_policy`** block in -`config.toml`: - -```toml -[shell_environment_policy] -# inherit can be "core" (default), "all", or "none" -inherit = "core" -# set to true to *skip* the filter for `"*KEY*"` and `"*TOKEN*"` -ignore_default_excludes = false -# exclude patterns (case-insensitive globs) -exclude = ["AWS_*", "AZURE_*"] -# force-set / override values -set = { CI = "1" } -# if provided, *only* vars matching these patterns are kept -include_only = ["PATH", "HOME"] -``` - -| Field | Type | Default | Description | -| ------------------------- | -------------------------- | ------- | ----------------------------------------------------------------------------------------------------------------------------------------------- | -| `inherit` | string | `core` | Starting template for the environment:
`core` (`HOME`, `PATH`, `USER`, …), `all` (clone full parent env), or `none` (start empty). | -| `ignore_default_excludes` | boolean | `false` | When `false`, Codex removes any var whose **name** contains `KEY`, `SECRET`, or `TOKEN` (case-insensitive) before other rules run. | -| `exclude` | array<string> | `[]` | Case-insensitive glob patterns to drop after the default filter.
Examples: `"AWS_*"`, `"AZURE_*"`. | -| `set` | table<string,string> | `{}` | Explicit key/value overrides or additions – always win over inherited values. | -| `include_only` | array<string> | `[]` | If non-empty, a whitelist of patterns; only variables that match _one_ pattern survive the final step. (Generally used with `inherit = "all"`.) | - -The patterns are **glob style**, not full regular expressions: `*` matches any -number of characters, `?` matches exactly one, and character classes like -`[A-Z]`/`[^0-9]` are supported. Matching is always **case-insensitive**. This -syntax is documented in code as `EnvironmentVariablePattern` (see -`core/src/config_types.rs`). - -If you just need a clean slate with a few custom entries you can write: - -```toml -[shell_environment_policy] -inherit = "none" -set = { PATH = "/usr/bin", MY_FLAG = "1" } -``` - -Currently, `CODEX_SANDBOX_NETWORK_DISABLED=1` is also added to the environment, assuming network is disabled. This is not configurable. - -### notify - -Specify a program that will be executed to get notified about events generated by Codex. Note that the program will receive the notification argument as a string of JSON, e.g.: - -```json -{ - "type": "agent-turn-complete", - "turn-id": "12345", - "input-messages": ["Rename `foo` to `bar` and update the callsites."], - "last-assistant-message": "Rename complete and verified `cargo build` succeeds." -} -``` - -The `"type"` property will always be set. Currently, `"agent-turn-complete"` is the only notification type that is supported. - -As an example, here is a Python script that parses the JSON and decides whether to show a desktop push notification using [terminal-notifier](https://github.com/julienXX/terminal-notifier) on macOS: - -```python -#!/usr/bin/env python3 - -import json -import subprocess -import sys - - -def main() -> int: - if len(sys.argv) != 2: - print("Usage: notify.py ") - return 1 - - try: - notification = json.loads(sys.argv[1]) - except json.JSONDecodeError: - return 1 - - match notification_type := notification.get("type"): - case "agent-turn-complete": - assistant_message = notification.get("last-assistant-message") - if assistant_message: - title = f"Codex: {assistant_message}" - else: - title = "Codex: Turn Complete!" - input_messages = notification.get("input_messages", []) - message = " ".join(input_messages) - title += message - case _: - print(f"not sending a push notification for: {notification_type}") - return 0 - - subprocess.check_output( - [ - "terminal-notifier", - "-title", - title, - "-message", - message, - "-group", - "codex", - "-ignoreDnD", - "-activate", - "com.googlecode.iterm2", - ] - ) - - return 0 - - -if __name__ == "__main__": - sys.exit(main()) -``` - -To have Codex use this script for notifications, you would configure it via `notify` in `~/.codex/config.toml` using the appropriate path to `notify.py` on your computer: +## Config -```toml -notify = ["python3", "/Users/mbolin/.codex/notify.py"] -``` +Codex supports a rich set of configuration options. See [`config.md`](./config.md) for details. -### history +## Model Context Protocol Support -By default, Codex CLI records messages sent to the model in `$CODEX_HOME/history.jsonl`. Note that on UNIX, the file permissions are set to `o600`, so it should only be readable and writable by the owner. +Codex CLI functions as an MCP client that can connect to MCP servers on startup. See the [`mcp_servers`](./config.md#mcp_servers) section in the configuration documentation for details. -To disable this behavior, configure `[history]` as follows: +It is still experimental, but you can also launch Codex as an MCP _server_ by running `codex mcp`. Using the [`@modelcontextprotocol/inspector`](https://github.com/modelcontextprotocol/inspector) is -```toml -[history] -persistence = "none" # "save-all" is the default value +```shell +npx @modelcontextprotocol/inspector codex mcp ``` -### file_opener - -Identifies the editor/URI scheme to use for hyperlinking citations in model output. If set, citations to files in the model output will be hyperlinked using the specified URI scheme so they can be ctrl/cmd-clicked from the terminal to open them. - -For example, if the model output includes a reference such as `【F:/home/user/project/main.py†L42-L50】`, then this would be rewritten to link to the URI `vscode://file/home/user/project/main.py:42`. - -Note this is **not** a general editor setting (like `$EDITOR`), as it only accepts a fixed set of values: - -- `"vscode"` (default) -- `"vscode-insiders"` -- `"windsurf"` -- `"cursor"` -- `"none"` to explicitly disable this feature - -Currently, `"vscode"` is the default, though Codex does not verify VS Code is installed. As such, `file_opener` may default to `"none"` or something else in the future. - -### project_doc_max_bytes - -Maximum number of bytes to read from an `AGENTS.md` file to include in the instructions sent with the first turn of a session. Defaults to 32 KiB. - -### tui +## Code Organization -Options that are specific to the TUI. +This folder is the root of a Cargo workspace. It contains quite a bit of experimental code, but here are the key crates: -```toml -[tui] -# This will make it so that Codex does not try to process mouse events, which -# means your Terminal's native drag-to-text to text selection and copy/paste -# should work. The tradeoff is that Codex will not receive any mouse events, so -# it will not be possible to use the mouse to scroll conversation history. -# -# Note that most terminals support holding down a modifier key when using the -# mouse to support text selection. For example, even if Codex mouse capture is -# enabled (i.e., this is set to `false`), you can still hold down alt while -# dragging the mouse to select text. -disable_mouse_capture = true # defaults to `false` -``` +- [`core/`](./core) contains the business logic for Codex. Ultimately, we hope this to be a library crate that is generally useful for building other Rust/native applications that use Codex. +- [`exec/`](./exec) "headless" CLI for use in automation. +- [`tui/`](./tui) CLI that launches a fullscreen TUI built with [Ratatui](https://ratatui.rs/). +- [`cli/`](./cli) CLI multitool that provides the aforementioned CLIs via subcommands. diff --git a/codex-rs/config.md b/codex-rs/config.md new file mode 100644 index 0000000000..a1caacfcbc --- /dev/null +++ b/codex-rs/config.md @@ -0,0 +1,377 @@ +# Config + +Codex supports several mechanisms for setting config values: + +- Config-specific command-line flags, such as `--model o3` (highest precedence). +- A generic `-c`/`--config` flag that takes a `key=value` pair, such as `--config model="o3"`. + - The key can contain dots to set a value deeper than the root, e.g. `--config model_providers.openai.wire_api="chat"`. + - Values can contain objects, such as `--config shell_environment_policy.include_only=["PATH", "HOME", "USER"]`. + - For consistency with `config.toml`, values are in TOML format rather than JSON format, so use `{a = 1, b = 2}` rather than `{"a": 1, "b": 2}`. + - If `value` cannot be parsed as a valid TOML value, it is treated as a string value. This means that both `-c model="o3"` and `-c model=o3` are equivalent. +- The `$CODEX_HOME/config.toml` configuration file where the `CODEX_HOME` environment value defaults to `~/.codex`. (Note `CODEX_HOME` will also be where logs and other Codex-related information are stored.) + +Both the `--config` flag and the `config.toml` file support the following options: + +## model + +The model that Codex should use. + +```toml +model = "o3" # overrides the default of "codex-mini-latest" +``` + +## model_provider + +Codex comes bundled with a number of "model providers" predefined. This config value is a string that indicates which provider to use. You can also define your own providers via `model_providers`. + +For example, if you are running ollama with Mistral locally, then you would need to add the following to your config: + +```toml +model = "mistral" +model_provider = "ollama" +``` + +because the following definition for `ollama` is included in Codex: + +```toml +[model_providers.ollama] +name = "Ollama" +base_url = "http://localhost:11434/v1" +wire_api = "chat" +``` + +This option defaults to `"openai"` and the corresponding provider is defined as follows: + +```toml +[model_providers.openai] +name = "OpenAI" +base_url = "https://api.openai.com/v1" +env_key = "OPENAI_API_KEY" +wire_api = "responses" +``` + +## model_providers + +This option lets you override and amend the default set of model providers bundled with Codex. This value is a map where the key is the value to use with `model_provider` to select the correspodning provider. + +For example, if you wanted to add a provider that uses the OpenAI 4o model via the chat completions API, then you + +```toml +# Recall that in TOML, root keys must be listed before tables. +model = "gpt-4o" +model_provider = "openai-chat-completions" + +[model_providers.openai-chat-completions] +# Name of the provider that will be displayed in the Codex UI. +name = "OpenAI using Chat Completions" +# The path `/chat/completions` will be amended to this URL to make the POST +# request for the chat completions. +base_url = "https://api.openai.com/v1" +# If `env_key` is set, identifies an environment variable that must be set when +# using Codex with this provider. The value of the environment variable must be +# non-empty and will be used in the `Bearer TOKEN` HTTP header for the POST request. +env_key = "OPENAI_API_KEY" +# valid values for wire_api are "chat" and "responses". +wire_api = "chat" +``` + +## approval_policy + +Determines when the user should be prompted to approve whether Codex can execute a command: + +```toml +# This is analogous to --suggest in the TypeScript Codex CLI +approval_policy = "unless-allow-listed" +``` + +```toml +# If the command fails when run in the sandbox, Codex asks for permission to +# retry the command outside the sandbox. +approval_policy = "on-failure" +``` + +```toml +# User is never prompted: if the command fails, Codex will automatically try +# something out. Note the `exec` subcommand always uses this mode. +approval_policy = "never" +``` + +## profiles + +A _profile_ is a collection of configuration values that can be set together. Multiple profiles can be defined in `config.toml` and you can specify the one you +want to use at runtime via the `--profile` flag. + +Here is an example of a `config.toml` that defines multiple profiles: + +```toml +model = "o3" +approval_policy = "unless-allow-listed" +sandbox_permissions = ["disk-full-read-access"] +disable_response_storage = false + +# Setting `profile` is equivalent to specifying `--profile o3` on the command +# line, though the `--profile` flag can still be used to override this value. +profile = "o3" + +[model_providers.openai-chat-completions] +name = "OpenAI using Chat Completions" +base_url = "https://api.openai.com/v1" +env_key = "OPENAI_API_KEY" +wire_api = "chat" + +[profiles.o3] +model = "o3" +model_provider = "openai" +approval_policy = "never" + +[profiles.gpt3] +model = "gpt-3.5-turbo" +model_provider = "openai-chat-completions" + +[profiles.zdr] +model = "o3" +model_provider = "openai" +approval_policy = "on-failure" +disable_response_storage = true +``` + +Users can specify config values at multiple levels. Order of precedence is as follows: + +1. custom command-line argument, e.g., `--model o3` +2. as part of a profile, where the `--profile` is specified via a CLI (or in the config file itself) +3. as an entry in `config.toml`, e.g., `model = "o3"` +4. the default value that comes with Codex CLI (i.e., Codex CLI defaults to `codex-mini-latest`) + +## sandbox_permissions + +List of permissions to grant to the sandbox that Codex uses to execute untrusted commands: + +```toml +# This is comparable to --full-auto in the TypeScript Codex CLI, though +# specifying `disk-write-platform-global-temp-folder` adds /tmp as a writable +# folder in addition to $TMPDIR. +sandbox_permissions = [ + "disk-full-read-access", + "disk-write-platform-user-temp-folder", + "disk-write-platform-global-temp-folder", + "disk-write-cwd", +] +``` + +To add additional writable folders, use `disk-write-folder`, which takes a parameter (this can be specified multiple times): + +```toml +sandbox_permissions = [ + # ... + "disk-write-folder=/Users/mbolin/.pyenv/shims", +] +``` + +## mcp_servers + +Defines the list of MCP servers that Codex can consult for tool use. Currently, only servers that are launched by executing a program that communicate over stdio are supported. For servers that use the SSE transport, consider an adapter like [mcp-proxy](https://github.com/sparfenyuk/mcp-proxy). + +**Note:** Codex may cache the list of tools and resources from an MCP server so that Codex can include this information in context at startup without spawning all the servers. This is designed to save resources by loading MCP servers lazily. + +This config option is comparable to how Claude and Cursor define `mcpServers` in their respective JSON config files, though because Codex uses TOML for its config language, the format is slightly different. For example, the following config in JSON: + +```json +{ + "mcpServers": { + "server-name": { + "command": "npx", + "args": ["-y", "mcp-server"], + "env": { + "API_KEY": "value" + } + } + } +} +``` + +Should be represented as follows in `~/.codex/config.toml`: + +```toml +# IMPORTANT: the top-level key is `mcp_servers` rather than `mcpServers`. +[mcp_servers.server-name] +command = "npx" +args = ["-y", "mcp-server"] +env = { "API_KEY" = "value" } +``` + +## disable_response_storage + +Currently, customers whose accounts are set to use Zero Data Retention (ZDR) must set `disable_response_storage` to `true` so that Codex uses an alternative to the Responses API that works with ZDR: + +```toml +disable_response_storage = true +``` + +## shell_environment_policy + +Codex spawns subprocesses (e.g. when executing a `local_shell` tool-call suggested by the assistant). By default it passes **only a minimal core subset** of your environment to those subprocesses to avoid leaking credentials. You can tune this behavior via the **`shell_environment_policy`** block in +`config.toml`: + +```toml +[shell_environment_policy] +# inherit can be "core" (default), "all", or "none" +inherit = "core" +# set to true to *skip* the filter for `"*KEY*"` and `"*TOKEN*"` +ignore_default_excludes = false +# exclude patterns (case-insensitive globs) +exclude = ["AWS_*", "AZURE_*"] +# force-set / override values +set = { CI = "1" } +# if provided, *only* vars matching these patterns are kept +include_only = ["PATH", "HOME"] +``` + +| Field | Type | Default | Description | +| ------------------------- | -------------------------- | ------- | ----------------------------------------------------------------------------------------------------------------------------------------------- | +| `inherit` | string | `core` | Starting template for the environment:
`core` (`HOME`, `PATH`, `USER`, …), `all` (clone full parent env), or `none` (start empty). | +| `ignore_default_excludes` | boolean | `false` | When `false`, Codex removes any var whose **name** contains `KEY`, `SECRET`, or `TOKEN` (case-insensitive) before other rules run. | +| `exclude` | array<string> | `[]` | Case-insensitive glob patterns to drop after the default filter.
Examples: `"AWS_*"`, `"AZURE_*"`. | +| `set` | table<string,string> | `{}` | Explicit key/value overrides or additions – always win over inherited values. | +| `include_only` | array<string> | `[]` | If non-empty, a whitelist of patterns; only variables that match _one_ pattern survive the final step. (Generally used with `inherit = "all"`.) | + +The patterns are **glob style**, not full regular expressions: `*` matches any +number of characters, `?` matches exactly one, and character classes like +`[A-Z]`/`[^0-9]` are supported. Matching is always **case-insensitive**. This +syntax is documented in code as `EnvironmentVariablePattern` (see +`core/src/config_types.rs`). + +If you just need a clean slate with a few custom entries you can write: + +```toml +[shell_environment_policy] +inherit = "none" +set = { PATH = "/usr/bin", MY_FLAG = "1" } +``` + +Currently, `CODEX_SANDBOX_NETWORK_DISABLED=1` is also added to the environment, assuming network is disabled. This is not configurable. + +## notify + +Specify a program that will be executed to get notified about events generated by Codex. Note that the program will receive the notification argument as a string of JSON, e.g.: + +```json +{ + "type": "agent-turn-complete", + "turn-id": "12345", + "input-messages": ["Rename `foo` to `bar` and update the callsites."], + "last-assistant-message": "Rename complete and verified `cargo build` succeeds." +} +``` + +The `"type"` property will always be set. Currently, `"agent-turn-complete"` is the only notification type that is supported. + +As an example, here is a Python script that parses the JSON and decides whether to show a desktop push notification using [terminal-notifier](https://github.com/julienXX/terminal-notifier) on macOS: + +```python +#!/usr/bin/env python3 + +import json +import subprocess +import sys + + +def main() -> int: + if len(sys.argv) != 2: + print("Usage: notify.py ") + return 1 + + try: + notification = json.loads(sys.argv[1]) + except json.JSONDecodeError: + return 1 + + match notification_type := notification.get("type"): + case "agent-turn-complete": + assistant_message = notification.get("last-assistant-message") + if assistant_message: + title = f"Codex: {assistant_message}" + else: + title = "Codex: Turn Complete!" + input_messages = notification.get("input_messages", []) + message = " ".join(input_messages) + title += message + case _: + print(f"not sending a push notification for: {notification_type}") + return 0 + + subprocess.check_output( + [ + "terminal-notifier", + "-title", + title, + "-message", + message, + "-group", + "codex", + "-ignoreDnD", + "-activate", + "com.googlecode.iterm2", + ] + ) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) +``` + +To have Codex use this script for notifications, you would configure it via `notify` in `~/.codex/config.toml` using the appropriate path to `notify.py` on your computer: + +```toml +notify = ["python3", "/Users/mbolin/.codex/notify.py"] +``` + +## history + +By default, Codex CLI records messages sent to the model in `$CODEX_HOME/history.jsonl`. Note that on UNIX, the file permissions are set to `o600`, so it should only be readable and writable by the owner. + +To disable this behavior, configure `[history]` as follows: + +```toml +[history] +persistence = "none" # "save-all" is the default value +``` + +## file_opener + +Identifies the editor/URI scheme to use for hyperlinking citations in model output. If set, citations to files in the model output will be hyperlinked using the specified URI scheme so they can be ctrl/cmd-clicked from the terminal to open them. + +For example, if the model output includes a reference such as `【F:/home/user/project/main.py†L42-L50】`, then this would be rewritten to link to the URI `vscode://file/home/user/project/main.py:42`. + +Note this is **not** a general editor setting (like `$EDITOR`), as it only accepts a fixed set of values: + +- `"vscode"` (default) +- `"vscode-insiders"` +- `"windsurf"` +- `"cursor"` +- `"none"` to explicitly disable this feature + +Currently, `"vscode"` is the default, though Codex does not verify VS Code is installed. As such, `file_opener` may default to `"none"` or something else in the future. + +## project_doc_max_bytes + +Maximum number of bytes to read from an `AGENTS.md` file to include in the instructions sent with the first turn of a session. Defaults to 32 KiB. + +## tui + +Options that are specific to the TUI. + +```toml +[tui] +# This will make it so that Codex does not try to process mouse events, which +# means your Terminal's native drag-to-text to text selection and copy/paste +# should work. The tradeoff is that Codex will not receive any mouse events, so +# it will not be possible to use the mouse to scroll conversation history. +# +# Note that most terminals support holding down a modifier key when using the +# mouse to support text selection. For example, even if Codex mouse capture is +# enabled (i.e., this is set to `false`), you can still hold down alt while +# dragging the mouse to select text. +disable_mouse_capture = true # defaults to `false` +``` From a0239c3cd63d782b43037b238332e3195e7e1cd4 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Fri, 30 May 2025 09:11:53 -0700 Subject: [PATCH 11/41] fix: enable `set positional-arguments` in justfile (#1169) The way these definitions worked before, they did not handle quoted args with spaces properly. For example, if you had `/tmp/test-just/printlen.py` as: ```python #!/usr/bin/env python3 import sys print(len(sys.argv)) ``` and your `justfile` was: ``` printlen *args: /tmp/test-just/printlen.py {{args}} ``` Then: ```shell $ just printlen foo bar 3 $ just printlen 'foo bar' 3 ``` which is not what we want: `'foo bar'` should be treated as one argument. The fix is to use [positional-arguments](https://github.com/casey/just/blob/515e806b5121a4696113ef15b5f0b12e69854570/README.md#L1131): ``` set positional-arguments printlen *args: /tmp/test-just/printlen.py "$@" ``` --- codex-rs/justfile | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/codex-rs/justfile b/codex-rs/justfile index 12088585ff..c09465a482 100644 --- a/codex-rs/justfile +++ b/codex-rs/justfile @@ -1,18 +1,20 @@ +set positional-arguments + # Display help help: just -l # `codex` codex *args: - cargo run --bin codex -- {{args}} + cargo run --bin codex -- "$@" # `codex exec` exec *args: - cargo run --bin codex -- exec {{args}} + cargo run --bin codex -- exec "$@" # `codex tui` tui *args: - cargo run --bin codex -- tui {{args}} + cargo run --bin codex -- tui "$@" # format code fmt: From baa92f37e0fe1cb80e5290d1e02ca9b6d3edd909 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Fri, 30 May 2025 10:55:28 -0700 Subject: [PATCH 12/41] feat: initial import of experimental GitHub Action (#1170) This is a first cut at a GitHub Action that lets you define prompt templates in `.md` files under `.github/codex/labels` that will run Codex with the associated prompt when the label is added to a GitHub pull request. For example, this PR includes these files: ``` .github/codex/labels/codex-attempt.md .github/codex/labels/codex-code-review.md .github/codex/labels/codex-investigate-issue.md ``` And the new `.github/workflows/codex.yml` workflow declares the following triggers: ```yaml on: issues: types: [opened, labeled] pull_request: branches: [main] types: [labeled] ``` as well as the following expression to gate the action: ``` jobs: codex: if: | (github.event_name == 'issues' && ( (github.event.action == 'labeled' && (github.event.label.name == 'codex-attempt' || github.event.label.name == 'codex-investigate-issue')) )) || (github.event_name == 'pull_request' && github.event.action == 'labeled' && github.event.label.name == 'codex-code-review') ``` Note the "actor" who added the label must have write access to the repo for the action to take effect. After adding a label, the action will "ack" the request by replacing the original label (e.g., `codex-review`) with an `-in-progress` suffix (e.g., `codex-review-in-progress`). When it is finished, it will swap the `-in-progress` label with a `-completed` one (e.g., `codex-review-completed`). Users of the action are responsible for providing an `OPENAI_API_KEY` and making it available as a secret to the action. --- .github/actions/codex/.gitignore | 1 + .github/actions/codex/.prettierrc.toml | 8 + .github/actions/codex/README.md | 140 +++++++++ .github/actions/codex/action.yml | 124 ++++++++ .github/actions/codex/bun.lock | 85 ++++++ .github/actions/codex/package.json | 21 ++ .github/actions/codex/src/add-reaction.ts | 85 ++++++ .github/actions/codex/src/comment.ts | 53 ++++ .github/actions/codex/src/config.ts | 11 + .../actions/codex/src/default-label-config.ts | 44 +++ .github/actions/codex/src/env-context.ts | 116 +++++++ .github/actions/codex/src/fail.ts | 4 + .github/actions/codex/src/git-helpers.ts | 149 +++++++++ .github/actions/codex/src/git-user.ts | 16 + .github/actions/codex/src/github-workspace.ts | 11 + .github/actions/codex/src/load-config.ts | 56 ++++ .github/actions/codex/src/main.ts | 80 +++++ .github/actions/codex/src/post-comment.ts | 60 ++++ .github/actions/codex/src/process-label.ts | 195 ++++++++++++ .github/actions/codex/src/prompt-template.ts | 284 ++++++++++++++++++ .github/actions/codex/src/review.ts | 42 +++ .github/actions/codex/src/run-codex.ts | 56 ++++ .github/actions/codex/src/verify-inputs.ts | 33 ++ .github/actions/codex/tsconfig.json | 15 + .github/codex/home/config.toml | 3 + .github/codex/labels/codex-attempt.md | 9 + .github/codex/labels/codex-review.md | 7 + .github/codex/labels/codex-triage.md | 7 + .github/workflows/codex.yml | 76 +++++ 29 files changed, 1791 insertions(+) create mode 100644 .github/actions/codex/.gitignore create mode 100644 .github/actions/codex/.prettierrc.toml create mode 100644 .github/actions/codex/README.md create mode 100644 .github/actions/codex/action.yml create mode 100644 .github/actions/codex/bun.lock create mode 100644 .github/actions/codex/package.json create mode 100644 .github/actions/codex/src/add-reaction.ts create mode 100644 .github/actions/codex/src/comment.ts create mode 100644 .github/actions/codex/src/config.ts create mode 100644 .github/actions/codex/src/default-label-config.ts create mode 100644 .github/actions/codex/src/env-context.ts create mode 100644 .github/actions/codex/src/fail.ts create mode 100644 .github/actions/codex/src/git-helpers.ts create mode 100644 .github/actions/codex/src/git-user.ts create mode 100644 .github/actions/codex/src/github-workspace.ts create mode 100644 .github/actions/codex/src/load-config.ts create mode 100755 .github/actions/codex/src/main.ts create mode 100644 .github/actions/codex/src/post-comment.ts create mode 100644 .github/actions/codex/src/process-label.ts create mode 100644 .github/actions/codex/src/prompt-template.ts create mode 100644 .github/actions/codex/src/review.ts create mode 100644 .github/actions/codex/src/run-codex.ts create mode 100644 .github/actions/codex/src/verify-inputs.ts create mode 100644 .github/actions/codex/tsconfig.json create mode 100644 .github/codex/home/config.toml create mode 100644 .github/codex/labels/codex-attempt.md create mode 100644 .github/codex/labels/codex-review.md create mode 100644 .github/codex/labels/codex-triage.md create mode 100644 .github/workflows/codex.yml diff --git a/.github/actions/codex/.gitignore b/.github/actions/codex/.gitignore new file mode 100644 index 0000000000..2ccbe4656c --- /dev/null +++ b/.github/actions/codex/.gitignore @@ -0,0 +1 @@ +/node_modules/ diff --git a/.github/actions/codex/.prettierrc.toml b/.github/actions/codex/.prettierrc.toml new file mode 100644 index 0000000000..4c58c583e5 --- /dev/null +++ b/.github/actions/codex/.prettierrc.toml @@ -0,0 +1,8 @@ +printWidth = 80 +quoteProps = "consistent" +semi = true +tabWidth = 2 +trailingComma = "all" + +# Preserve existing behavior for markdown/text wrapping. +proseWrap = "preserve" diff --git a/.github/actions/codex/README.md b/.github/actions/codex/README.md new file mode 100644 index 0000000000..a0be8ecb68 --- /dev/null +++ b/.github/actions/codex/README.md @@ -0,0 +1,140 @@ +# openai/codex-action + +`openai/codex-action` is a GitHub Action that facilitates the use of [Codex](https://github.com/openai/codex) on GitHub issues and pull requests. Using the action, associate **labels** to run Codex with the appropriate prompt for the given context. Codex will respond by posting comments or creating PRs, whichever you specify! + +Here is a sample workflow that uses `openai/codex-action`: + +```yaml +name: Codex + +on: + issues: + types: [opened, labeled] + pull_request: + branches: [main] + types: [labeled] + +jobs: + codex: + if: ... # optional, but can be effective in conserving CI resources + runs-on: ubuntu-latest + # TODO(mbolin): Need to verify if/when `write` is necessary. + permissions: + contents: write + issues: write + pull-requests: write + steps: + # By default, Codex runs network disabled using --full-auto, so perform + # any setup that requires network (such as installing dependencies) + # before openai/codex-action. + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Run Codex + uses: openai/codex-action@latest + with: + openai_api_key: ${{ secrets.CODEX_OPENAI_API_KEY }} + github_token: ${{ secrets.GITHUB_TOKEN }} +``` + +See sample usage in [`codex.yml`](../../workflows/codex.yml). + +## Triggering the Action + +Using the sample workflow above, we have: + +```yaml +on: + issues: + types: [opened, labeled] + pull_request: + branches: [main] + types: [labeled] +``` + +which means our workflow will be triggered when any of the following events occur: + +- a label is added to an issue +- a label is added to a pull request against the `main` branch + +### Label-Based Triggers + +To define a GitHub label that should trigger Codex, create a file named `.github/codex/labels/LABEL-NAME.md` in your repository where `LABEL-NAME` is the name of the label. The content of the file is the prompt template to use when the label is added (see more on [Prompt Template Variables](#prompt-template-variables) below). + +For example, if the file `.github/codex/labels/codex-review.md` exists, then: + +- Adding the `codex-review` label will trigger the workflow containing the `openai/codex-action` GitHub Action. +- When `openai/codex-action` starts, it will replace the `codex-review` label with `codex-review-in-progress`. +- When `openai/codex-action` is finished, it will replace the `codex-review-in-progress` label with `codex-review-completed`. + +If Codex sees that either `codex-review-in-progress` or `codex-review-completed` is already present, it will not perform the action. + +As determined by the [default config](./src/default-label-config.ts), Codex will act on the following labels by default: + +- Adding the `codex-review` label to a pull request will have Codex review the PR and add it to the PR as a comment. +- Adding the `codex-triage` label to an issue will have Codex investigate the issue and report its findings as a comment. +- Adding the `codex-issue-fix` label to an issue will have Codex attempt to fix the issue and create a PR wit the fix, if any. + +## Action Inputs + +The `openai/codex-action` GitHub Action takes the following inputs + +### `openai_api_key` (required) + +Set your `OPENAI_API_KEY` as a [repository secret](https://docs.github.com/en/actions/security-for-github-actions/security-guides/using-secrets-in-github-actions). See **Secrets and varaibles** then **Actions** in the settings for your GitHub repo. + +Note that the secret name does not have to be `OPENAI_API_KEY`. For example, you might want to name it `CODEX_OPENAI_API_KEY` and then configure it on `openai/codex-action` as follows: + +```yaml +openai_api_key: ${{ secrets.CODEX_OPENAI_API_KEY }} +``` + +### `github_token` (required) + +This is required so that Codex can post a comment or create a PR. Set this value on the action as follows: + +```yaml +github_token: ${{ secrets.GITHUB_TOKEN }} +``` + +### `codex_args` + +A whitespace-delimited list of arguments to pass to Codex. Defaults to `--full-auto`, but if you want to override the default model to use `o3`: + +```yaml +codex_args: "--full-auto --model o3" +``` + +For more complex configurations, use the `codex_home` input. + +### `codex_home` + +If set, the value to use for the `$CODEX_HOME` environment variable when running Codex. As explained [in the docs](https://github.com/openai/codex/tree/main/codex-rs#readme), this folder can contain the `config.toml` to configure Codex, custom instructions, and log files. + +This should be a relative path within your repo. + +## Prompt Template Variables + +As shown above, `"prompt"` and `"promptPath"` are used to define prompt templates that will be populated and passed to Codex in response to certain events. All template variables are of the form `{CODEX_ACTION_...}` and the supported values are defined below. + +### `CODEX_ACTION_ISSUE_TITLE` + +If the action was triggered on a GitHub issue, this is the issue title. + +Specifically it is read as the `.issue.title` from the `$GITHUB_EVENT_PATH`. + +### `CODEX_ACTION_ISSUE_BODY` + +If the action was triggered on a GitHub issue, this is the issue body. + +Specifically it is read as the `.issue.body` from the `$GITHUB_EVENT_PATH`. + +### `CODEX_ACTION_GITHUB_EVENT_PATH` + +The value of the `$GITHUB_EVENT_PATH` environment variable, which is the path to the file that contains the JSON payload for the event that triggered the workflow. Codex can use `jq` to read only the fields of interest from this file. + +### `CODEX_ACTION_PR_DIFF` + +If the action was triggered on a pull request, this is the diff between the base and head commits of the PR. It is the output from `git diff`. + +Note that the content of the diff could be quite large, so is generally safer to point Codex at `CODEX_ACTION_GITHUB_EVENT_PATH` and let it decide how it wants to explore the change. diff --git a/.github/actions/codex/action.yml b/.github/actions/codex/action.yml new file mode 100644 index 0000000000..715423d06a --- /dev/null +++ b/.github/actions/codex/action.yml @@ -0,0 +1,124 @@ +name: "Codex [reusable action]" +description: "A reusable action that runs a Codex model." + +inputs: + openai_api_key: + description: "The value to use as the OPENAI_API_KEY environment variable when running Codex." + required: true + trigger_phrase: + description: "Text to trigger Codex from a PR/issue body or comment." + required: false + default: "" + github_token: + description: "Token so Codex can comment on the PR or issue." + required: true + codex_args: + description: "A whitespace-delimited list of arguments to pass to Codex. Due to limitations in YAML, arguments with spaces are not supported. For more complex configurations, use the `codex_home` input." + required: false + default: "--full-auto" + codex_home: + description: "Value to use as the CODEX_HOME environment variable when running Codex." + required: false + codex_release_tag: + description: "The release tag of the Codex model to run." + required: false + default: "codex-rs-d519bd8bbd1e1fd9efdc5d68cf7bebdec0dd0f28-1-rust-v0.0.2505270918" + +runs: + using: "composite" + steps: + # Do this in Bash so we do not even bother to install Bun if the sender does + # not have write access to the repo. + - name: Verify user has write access to the repo. + env: + GH_TOKEN: ${{ github.token }} + shell: bash + run: | + set -euo pipefail + + PERMISSION=$(gh api \ + "/repos/${GITHUB_REPOSITORY}/collaborators/${{ github.event.sender.login }}/permission" \ + | jq -r '.permission') + + if [[ "$PERMISSION" != "admin" && "$PERMISSION" != "write" ]]; then + exit 1 + fi + + - name: Download Codex + env: + GH_TOKEN: ${{ github.token }} + shell: bash + run: | + set -euo pipefail + + # Determine OS/arch and corresponding Codex artifact name. + uname_s=$(uname -s) + uname_m=$(uname -m) + + case "$uname_s" in + Linux*) os="linux" ;; + Darwin*) os="apple-darwin" ;; + *) echo "Unsupported operating system: $uname_s"; exit 1 ;; + esac + + case "$uname_m" in + x86_64*) arch="x86_64" ;; + arm64*|aarch64*) arch="aarch64" ;; + *) echo "Unsupported architecture: $uname_m"; exit 1 ;; + esac + + # linux builds differentiate between musl and gnu. + if [[ "$os" == "linux" ]]; then + if [[ "$arch" == "x86_64" ]]; then + triple="${arch}-unknown-linux-musl" + else + # Only other supported linux build is aarch64 gnu. + triple="${arch}-unknown-linux-gnu" + fi + else + # macOS + triple="${arch}-apple-darwin" + fi + + # Note that if we start baking version numbers into the artifact name, + # we will need to update this action.yml file to match. + artifact="codex-exec-${triple}.tar.gz" + + gh release download ${{ inputs.codex_release_tag }} --repo openai/codex \ + --pattern "$artifact" --output - \ + | tar xzO > /usr/local/bin/codex-exec + chmod +x /usr/local/bin/codex-exec + + # Display Codex version to confirm binary integrity; ensure we point it + # at the checked-out repository via --cd so that any subsequent commands + # use the correct working directory. + codex-exec --cd "$GITHUB_WORKSPACE" --version + + - name: Install Bun + uses: oven-sh/setup-bun@v2 + with: + bun-version: 1.2.11 + + - name: Install dependencies + shell: bash + run: | + cd ${{ github.action_path }} + bun install --production + + - name: Run Codex + shell: bash + run: bun run ${{ github.action_path }}/src/main.ts + # Process args plus environment variables often have a max of 128 KiB, + # so we should fit within that limit? + env: + INPUT_CODEX_ARGS: ${{ inputs.codex_args || '' }} + INPUT_CODEX_HOME: ${{ inputs.codex_home || ''}} + INPUT_TRIGGER_PHRASE: ${{ inputs.trigger_phrase || '' }} + OPENAI_API_KEY: ${{ inputs.openai_api_key }} + GITHUB_TOKEN: ${{ inputs.github_token }} + GITHUB_EVENT_ACTION: ${{ github.event.action || '' }} + GITHUB_EVENT_LABEL_NAME: ${{ github.event.label.name || '' }} + GITHUB_EVENT_ISSUE_NUMBER: ${{ github.event.issue.number || '' }} + GITHUB_EVENT_ISSUE_BODY: ${{ github.event.issue.body || '' }} + GITHUB_EVENT_REVIEW_BODY: ${{ github.event.review.body || '' }} + GITHUB_EVENT_COMMENT_BODY: ${{ github.event.comment.body || '' }} diff --git a/.github/actions/codex/bun.lock b/.github/actions/codex/bun.lock new file mode 100644 index 0000000000..11b791654b --- /dev/null +++ b/.github/actions/codex/bun.lock @@ -0,0 +1,85 @@ +{ + "lockfileVersion": 1, + "workspaces": { + "": { + "name": "codex-action", + "dependencies": { + "@actions/core": "^1.11.1", + "@actions/github": "^6.0.1", + }, + "devDependencies": { + "@types/bun": "^1.2.11", + "@types/node": "^22.15.21", + "prettier": "^3.5.3", + "typescript": "^5.8.3", + }, + }, + }, + "packages": { + "@actions/core": ["@actions/core@1.11.1", "", { "dependencies": { "@actions/exec": "^1.1.1", "@actions/http-client": "^2.0.1" } }, "sha512-hXJCSrkwfA46Vd9Z3q4cpEpHB1rL5NG04+/rbqW9d3+CSvtB1tYe8UTpAlixa1vj0m/ULglfEK2UKxMGxCxv5A=="], + + "@actions/exec": ["@actions/exec@1.1.1", "", { "dependencies": { "@actions/io": "^1.0.1" } }, "sha512-+sCcHHbVdk93a0XT19ECtO/gIXoxvdsgQLzb2fE2/5sIZmWQuluYyjPQtrtTHdU1YzTZ7bAPN4sITq2xi1679w=="], + + "@actions/github": ["@actions/github@6.0.1", "", { "dependencies": { "@actions/http-client": "^2.2.0", "@octokit/core": "^5.0.1", "@octokit/plugin-paginate-rest": "^9.2.2", "@octokit/plugin-rest-endpoint-methods": "^10.4.0", "@octokit/request": "^8.4.1", "@octokit/request-error": "^5.1.1", "undici": "^5.28.5" } }, "sha512-xbZVcaqD4XnQAe35qSQqskb3SqIAfRyLBrHMd/8TuL7hJSz2QtbDwnNM8zWx4zO5l2fnGtseNE3MbEvD7BxVMw=="], + + "@actions/http-client": ["@actions/http-client@2.2.3", "", { "dependencies": { "tunnel": "^0.0.6", "undici": "^5.25.4" } }, "sha512-mx8hyJi/hjFvbPokCg4uRd4ZX78t+YyRPtnKWwIl+RzNaVuFpQHfmlGVfsKEJN8LwTCvL+DfVgAM04XaHkm6bA=="], + + "@actions/io": ["@actions/io@1.1.3", "", {}, "sha512-wi9JjgKLYS7U/z8PPbco+PvTb/nRWjeoFlJ1Qer83k/3C5PHQi28hiVdeE2kHXmIL99mQFawx8qt/JPjZilJ8Q=="], + + "@fastify/busboy": ["@fastify/busboy@2.1.1", "", {}, "sha512-vBZP4NlzfOlerQTnba4aqZoMhE/a9HY7HRqoOPaETQcSQuWEIyZMHGfVu6w9wGtGK5fED5qRs2DteVCjOH60sA=="], + + "@octokit/auth-token": ["@octokit/auth-token@4.0.0", "", {}, "sha512-tY/msAuJo6ARbK6SPIxZrPBms3xPbfwBrulZe0Wtr/DIY9lje2HeV1uoebShn6mx7SjCHif6EjMvoREj+gZ+SA=="], + + "@octokit/core": ["@octokit/core@5.2.1", "", { "dependencies": { "@octokit/auth-token": "^4.0.0", "@octokit/graphql": "^7.1.0", "@octokit/request": "^8.4.1", "@octokit/request-error": "^5.1.1", "@octokit/types": "^13.0.0", "before-after-hook": "^2.2.0", "universal-user-agent": "^6.0.0" } }, "sha512-dKYCMuPO1bmrpuogcjQ8z7ICCH3FP6WmxpwC03yjzGfZhj9fTJg6+bS1+UAplekbN2C+M61UNllGOOoAfGCrdQ=="], + + "@octokit/endpoint": ["@octokit/endpoint@9.0.6", "", { "dependencies": { "@octokit/types": "^13.1.0", "universal-user-agent": "^6.0.0" } }, "sha512-H1fNTMA57HbkFESSt3Y9+FBICv+0jFceJFPWDePYlR/iMGrwM5ph+Dd4XRQs+8X+PUFURLQgX9ChPfhJ/1uNQw=="], + + "@octokit/graphql": ["@octokit/graphql@7.1.1", "", { "dependencies": { "@octokit/request": "^8.4.1", "@octokit/types": "^13.0.0", "universal-user-agent": "^6.0.0" } }, "sha512-3mkDltSfcDUoa176nlGoA32RGjeWjl3K7F/BwHwRMJUW/IteSa4bnSV8p2ThNkcIcZU2umkZWxwETSSCJf2Q7g=="], + + "@octokit/openapi-types": ["@octokit/openapi-types@24.2.0", "", {}, "sha512-9sIH3nSUttelJSXUrmGzl7QUBFul0/mB8HRYl3fOlgHbIWG+WnYDXU3v/2zMtAvuzZ/ed00Ei6on975FhBfzrg=="], + + "@octokit/plugin-paginate-rest": ["@octokit/plugin-paginate-rest@9.2.2", "", { "dependencies": { "@octokit/types": "^12.6.0" }, "peerDependencies": { "@octokit/core": "5" } }, "sha512-u3KYkGF7GcZnSD/3UP0S7K5XUFT2FkOQdcfXZGZQPGv3lm4F2Xbf71lvjldr8c1H3nNbF+33cLEkWYbokGWqiQ=="], + + "@octokit/plugin-rest-endpoint-methods": ["@octokit/plugin-rest-endpoint-methods@10.4.1", "", { "dependencies": { "@octokit/types": "^12.6.0" }, "peerDependencies": { "@octokit/core": "5" } }, "sha512-xV1b+ceKV9KytQe3zCVqjg+8GTGfDYwaT1ATU5isiUyVtlVAO3HNdzpS4sr4GBx4hxQ46s7ITtZrAsxG22+rVg=="], + + "@octokit/request": ["@octokit/request@8.4.1", "", { "dependencies": { "@octokit/endpoint": "^9.0.6", "@octokit/request-error": "^5.1.1", "@octokit/types": "^13.1.0", "universal-user-agent": "^6.0.0" } }, "sha512-qnB2+SY3hkCmBxZsR/MPCybNmbJe4KAlfWErXq+rBKkQJlbjdJeS85VI9r8UqeLYLvnAenU8Q1okM/0MBsAGXw=="], + + "@octokit/request-error": ["@octokit/request-error@5.1.1", "", { "dependencies": { "@octokit/types": "^13.1.0", "deprecation": "^2.0.0", "once": "^1.4.0" } }, "sha512-v9iyEQJH6ZntoENr9/yXxjuezh4My67CBSu9r6Ve/05Iu5gNgnisNWOsoJHTP6k0Rr0+HQIpnH+kyammu90q/g=="], + + "@octokit/types": ["@octokit/types@13.10.0", "", { "dependencies": { "@octokit/openapi-types": "^24.2.0" } }, "sha512-ifLaO34EbbPj0Xgro4G5lP5asESjwHracYJvVaPIyXMuiuXLlhic3S47cBdTb+jfODkTE5YtGCLt3Ay3+J97sA=="], + + "@types/bun": ["@types/bun@1.2.13", "", { "dependencies": { "bun-types": "1.2.13" } }, "sha512-u6vXep/i9VBxoJl3GjZsl/BFIsvML8DfVDO0RYLEwtSZSp981kEO1V5NwRcO1CPJ7AmvpbnDCiMKo3JvbDEjAg=="], + + "@types/node": ["@types/node@22.15.21", "", { "dependencies": { "undici-types": "~6.21.0" } }, "sha512-EV/37Td6c+MgKAbkcLG6vqZ2zEYHD7bvSrzqqs2RIhbA6w3x+Dqz8MZM3sP6kGTeLrdoOgKZe+Xja7tUB2DNkQ=="], + + "before-after-hook": ["before-after-hook@2.2.3", "", {}, "sha512-NzUnlZexiaH/46WDhANlyR2bXRopNg4F/zuSA3OpZnllCUgRaOF2znDioDWrmbNVsuZk6l9pMquQB38cfBZwkQ=="], + + "bun-types": ["bun-types@1.2.13", "", { "dependencies": { "@types/node": "*" } }, "sha512-rRjA1T6n7wto4gxhAO/ErZEtOXyEZEmnIHQfl0Dt1QQSB4QV0iP6BZ9/YB5fZaHFQ2dwHFrmPaRQ9GGMX01k9Q=="], + + "deprecation": ["deprecation@2.3.1", "", {}, "sha512-xmHIy4F3scKVwMsQ4WnVaS8bHOx0DmVwRywosKhaILI0ywMDWPtBSku2HNxRvF7jtwDRsoEwYQSfbxj8b7RlJQ=="], + + "once": ["once@1.4.0", "", { "dependencies": { "wrappy": "1" } }, "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w=="], + + "prettier": ["prettier@3.5.3", "", { "bin": { "prettier": "bin/prettier.cjs" } }, "sha512-QQtaxnoDJeAkDvDKWCLiwIXkTgRhwYDEQCghU9Z6q03iyek/rxRh/2lC3HB7P8sWT2xC/y5JDctPLBIGzHKbhw=="], + + "tunnel": ["tunnel@0.0.6", "", {}, "sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg=="], + + "typescript": ["typescript@5.8.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-p1diW6TqL9L07nNxvRMM7hMMw4c5XOo/1ibL4aAIGmSAt9slTE1Xgw5KWuof2uTOvCg9BY7ZRi+GaF+7sfgPeQ=="], + + "undici": ["undici@5.29.0", "", { "dependencies": { "@fastify/busboy": "^2.0.0" } }, "sha512-raqeBD6NQK4SkWhQzeYKd1KmIG6dllBOTt55Rmkt4HtI9mwdWtJljnrXjAFUBLTSN67HWrOIZ3EPF4kjUw80Bg=="], + + "undici-types": ["undici-types@6.21.0", "", {}, "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ=="], + + "universal-user-agent": ["universal-user-agent@6.0.1", "", {}, "sha512-yCzhz6FN2wU1NiiQRogkTQszlQSlpWaw8SvVegAc+bDxbzHgh1vX8uIe8OYyMH6DwH+sdTJsgMl36+mSMdRJIQ=="], + + "wrappy": ["wrappy@1.0.2", "", {}, "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ=="], + + "@octokit/plugin-paginate-rest/@octokit/types": ["@octokit/types@12.6.0", "", { "dependencies": { "@octokit/openapi-types": "^20.0.0" } }, "sha512-1rhSOfRa6H9w4YwK0yrf5faDaDTb+yLyBUKOCV4xtCDB5VmIPqd/v9yr9o6SAzOAlRxMiRiCic6JVM1/kunVkw=="], + + "@octokit/plugin-rest-endpoint-methods/@octokit/types": ["@octokit/types@12.6.0", "", { "dependencies": { "@octokit/openapi-types": "^20.0.0" } }, "sha512-1rhSOfRa6H9w4YwK0yrf5faDaDTb+yLyBUKOCV4xtCDB5VmIPqd/v9yr9o6SAzOAlRxMiRiCic6JVM1/kunVkw=="], + + "@octokit/plugin-paginate-rest/@octokit/types/@octokit/openapi-types": ["@octokit/openapi-types@20.0.0", "", {}, "sha512-EtqRBEjp1dL/15V7WiX5LJMIxxkdiGJnabzYx5Apx4FkQIFgAfKumXeYAqqJCj1s+BMX4cPFIFC4OLCR6stlnA=="], + + "@octokit/plugin-rest-endpoint-methods/@octokit/types/@octokit/openapi-types": ["@octokit/openapi-types@20.0.0", "", {}, "sha512-EtqRBEjp1dL/15V7WiX5LJMIxxkdiGJnabzYx5Apx4FkQIFgAfKumXeYAqqJCj1s+BMX4cPFIFC4OLCR6stlnA=="], + } +} diff --git a/.github/actions/codex/package.json b/.github/actions/codex/package.json new file mode 100644 index 0000000000..bb35ee3a47 --- /dev/null +++ b/.github/actions/codex/package.json @@ -0,0 +1,21 @@ +{ + "name": "codex-action", + "version": "0.0.0", + "private": true, + "scripts": { + "format": "prettier --check src", + "format:fix": "prettier --write src", + "test": "bun test", + "typecheck": "tsc" + }, + "dependencies": { + "@actions/core": "^1.11.1", + "@actions/github": "^6.0.1" + }, + "devDependencies": { + "@types/bun": "^1.2.11", + "@types/node": "^22.15.21", + "prettier": "^3.5.3", + "typescript": "^5.8.3" + } +} diff --git a/.github/actions/codex/src/add-reaction.ts b/.github/actions/codex/src/add-reaction.ts new file mode 100644 index 0000000000..85026dd9af --- /dev/null +++ b/.github/actions/codex/src/add-reaction.ts @@ -0,0 +1,85 @@ +import * as github from "@actions/github"; +import type { EnvContext } from "./env-context"; + +/** + * Add an "eyes" reaction to the entity (issue, issue comment, or pull request + * review comment) that triggered the current Codex invocation. + * + * The purpose is to provide immediate feedback to the user – similar to the + * *-in-progress label flow – indicating that the bot has acknowledged the + * request and is working on it. + * + * We attempt to add the reaction best suited for the current GitHub event: + * + * • issues → POST /repos/{owner}/{repo}/issues/{issue_number}/reactions + * • issue_comment → POST /repos/{owner}/{repo}/issues/comments/{comment_id}/reactions + * • pull_request_review_comment → POST /repos/{owner}/{repo}/pulls/comments/{comment_id}/reactions + * + * If the specific target is unavailable (e.g. unexpected payload shape) we + * silently skip instead of failing the whole action because the reaction is + * merely cosmetic. + */ +export async function addEyesReaction(ctx: EnvContext): Promise { + const octokit = ctx.getOctokit(); + const { owner, repo } = github.context.repo; + const eventName = github.context.eventName; + + try { + switch (eventName) { + case "issue_comment": { + const commentId = (github.context.payload as any)?.comment?.id; + if (commentId) { + await octokit.rest.reactions.createForIssueComment({ + owner, + repo, + comment_id: commentId, + content: "eyes", + }); + return; + } + break; + } + case "pull_request_review_comment": { + const commentId = (github.context.payload as any)?.comment?.id; + if (commentId) { + await octokit.rest.reactions.createForPullRequestReviewComment({ + owner, + repo, + comment_id: commentId, + content: "eyes", + }); + return; + } + break; + } + case "issues": { + const issueNumber = github.context.issue.number; + if (issueNumber) { + await octokit.rest.reactions.createForIssue({ + owner, + repo, + issue_number: issueNumber, + content: "eyes", + }); + return; + } + break; + } + default: { + // Fallback: try to react to the issue/PR if we have a number. + const issueNumber = github.context.issue.number; + if (issueNumber) { + await octokit.rest.reactions.createForIssue({ + owner, + repo, + issue_number: issueNumber, + content: "eyes", + }); + } + } + } + } catch (error) { + // Do not fail the action if reaction creation fails – log and continue. + console.warn(`Failed to add \"eyes\" reaction: ${error}`); + } +} diff --git a/.github/actions/codex/src/comment.ts b/.github/actions/codex/src/comment.ts new file mode 100644 index 0000000000..6e2833aff0 --- /dev/null +++ b/.github/actions/codex/src/comment.ts @@ -0,0 +1,53 @@ +import type { EnvContext } from "./env-context"; +import { runCodex } from "./run-codex"; +import { postComment } from "./post-comment"; +import { addEyesReaction } from "./add-reaction"; + +/** + * Handle `issue_comment` and `pull_request_review_comment` events once we know + * the action is supported. + */ +export async function onComment(ctx: EnvContext): Promise { + const triggerPhrase = ctx.tryGet("INPUT_TRIGGER_PHRASE"); + if (!triggerPhrase) { + console.warn("Empty trigger phrase: skipping."); + return; + } + + // Attempt to get the body of the comment from the environment. Depending on + // the event type either `GITHUB_EVENT_COMMENT_BODY` (issue & PR comments) or + // `GITHUB_EVENT_REVIEW_BODY` (PR reviews) is set. + const commentBody = + ctx.tryGetNonEmpty("GITHUB_EVENT_COMMENT_BODY") ?? + ctx.tryGetNonEmpty("GITHUB_EVENT_REVIEW_BODY") ?? + ctx.tryGetNonEmpty("GITHUB_EVENT_ISSUE_BODY"); + + if (!commentBody) { + console.warn("Comment body not found in environment: skipping."); + return; + } + + // Check if the trigger phrase is present. + if (!commentBody.includes(triggerPhrase)) { + console.log( + `Trigger phrase '${triggerPhrase}' not found: nothing to do for this comment.`, + ); + return; + } + + // Derive the prompt by removing the trigger phrase. Remove only the first + // occurrence to keep any additional occurrences that might be meaningful. + const prompt = commentBody.replace(triggerPhrase, "").trim(); + + if (prompt.length === 0) { + console.warn("Prompt is empty after removing trigger phrase: skipping"); + return; + } + + // Provide immediate feedback that we are working on the request. + await addEyesReaction(ctx); + + // Run Codex and post the response as a new comment. + const lastMessage = await runCodex(prompt, ctx); + await postComment(lastMessage, ctx); +} diff --git a/.github/actions/codex/src/config.ts b/.github/actions/codex/src/config.ts new file mode 100644 index 0000000000..1f98f946ab --- /dev/null +++ b/.github/actions/codex/src/config.ts @@ -0,0 +1,11 @@ +import { readdirSync, statSync } from "fs"; +import * as path from "path"; + +export interface Config { + labels: Record; +} + +export interface LabelConfig { + /** Returns the prompt template. */ + getPromptTemplate(): string; +} diff --git a/.github/actions/codex/src/default-label-config.ts b/.github/actions/codex/src/default-label-config.ts new file mode 100644 index 0000000000..270f1f9c5d --- /dev/null +++ b/.github/actions/codex/src/default-label-config.ts @@ -0,0 +1,44 @@ +import type { Config } from "./config"; + +export function getDefaultConfig(): Config { + return { + labels: { + "codex-investigate-issue": { + getPromptTemplate: () => + ` +Troubleshoot whether the reported issue is valid. + +Provide a concise and respectful comment summarizing the findings. + +### {CODEX_ACTION_ISSUE_TITLE} + +{CODEX_ACTION_ISSUE_BODY} +`.trim(), + }, + "codex-code-review": { + getPromptTemplate: () => + ` +Review this PR and respond with a very concise final message, formatted in Markdown. + +There should be a summary of the changes (1-2 sentences) and a few bullet points if necessary. + +Then provide the **review** (1-2 sentences plus bullet points, friendly tone). + +{CODEX_ACTION_GITHUB_EVENT_PATH} contains the JSON that triggered this GitHub workflow. It contains the \`base\` and \`head\` refs that define this PR. Both refs are available locally. +`.trim(), + }, + "codex-attempt-fix": { + getPromptTemplate: () => + ` +Attempt to solve the reported issue. + +If a code change is required, create a new branch, commit the fix, and open a pull-request that resolves the problem. + +### {CODEX_ACTION_ISSUE_TITLE} + +{CODEX_ACTION_ISSUE_BODY} +`.trim(), + }, + }, + }; +} diff --git a/.github/actions/codex/src/env-context.ts b/.github/actions/codex/src/env-context.ts new file mode 100644 index 0000000000..9c18e0e6a2 --- /dev/null +++ b/.github/actions/codex/src/env-context.ts @@ -0,0 +1,116 @@ +/* + * Centralised access to environment variables used by the Codex GitHub + * Action. + * + * To enable proper unit-testing we avoid reading from `process.env` at module + * initialisation time. Instead a `EnvContext` object is created (usually from + * the real `process.env`) and passed around explicitly or – where that is not + * yet practical – imported as the shared `defaultContext` singleton. Tests can + * create their own context backed by a stubbed map of variables without having + * to mutate global state. + */ + +import { fail } from "./fail"; +import * as github from "@actions/github"; + +export interface EnvContext { + /** + * Return the value for a given environment variable or terminate the action + * via `fail` if it is missing / empty. + */ + get(name: string): string; + + /** + * Attempt to read an environment variable. Returns the value when present; + * otherwise returns undefined (does not call `fail`). + */ + tryGet(name: string): string | undefined; + + /** + * Attempt to read an environment variable. Returns non-empty string value or + * null if unset or empty string. + */ + tryGetNonEmpty(name: string): string | null; + + /** + * Return a memoised Octokit instance authenticated via the token resolved + * from the provided argument (when defined) or the environment variables + * `GITHUB_TOKEN`/`GH_TOKEN`. + * + * Subsequent calls return the same cached instance to avoid spawning + * multiple REST clients within a single action run. + */ + getOctokit(token?: string): ReturnType; +} + +/** Internal helper – *not* exported. */ +function _getRequiredEnv( + name: string, + env: Record, +): string | undefined { + const value = env[name]; + + // Avoid leaking secrets into logs while still logging non-secret variables. + if (name.endsWith("KEY") || name.endsWith("TOKEN")) { + if (value) { + console.log(`value for ${name} was found`); + } + } else { + console.log(`${name}=${value}`); + } + + return value; +} + +/** Create a context backed by the supplied environment map (defaults to `process.env`). */ +export function createEnvContext( + env: Record = process.env, +): EnvContext { + // Lazily instantiated Octokit client – shared across this context. + let cachedOctokit: ReturnType | null = null; + + return { + get(name: string): string { + const value = _getRequiredEnv(name, env); + if (value == null) { + fail(`Missing required environment variable: ${name}`); + } + return value; + }, + + tryGet(name: string): string | undefined { + return _getRequiredEnv(name, env); + }, + + tryGetNonEmpty(name: string): string | null { + const value = _getRequiredEnv(name, env); + return value == null || value === "" ? null : value; + }, + + getOctokit(token?: string) { + if (cachedOctokit) { + return cachedOctokit; + } + + // Determine the token to authenticate with. + const githubToken = token ?? env["GITHUB_TOKEN"] ?? env["GH_TOKEN"]; + + if (!githubToken) { + fail( + "Unable to locate a GitHub token. `github_token` should have been set on the action.", + ); + } + + cachedOctokit = github.getOctokit(githubToken!); + return cachedOctokit; + }, + }; +} + +/** + * Shared context built from the actual `process.env`. Production code that is + * not yet refactored to receive a context explicitly may import and use this + * singleton. Tests should avoid the singleton and instead pass their own + * context to the functions they exercise. + */ +export const defaultContext: EnvContext = createEnvContext(); diff --git a/.github/actions/codex/src/fail.ts b/.github/actions/codex/src/fail.ts new file mode 100644 index 0000000000..924d70095c --- /dev/null +++ b/.github/actions/codex/src/fail.ts @@ -0,0 +1,4 @@ +export function fail(message: string): never { + console.error(message); + process.exit(1); +} diff --git a/.github/actions/codex/src/git-helpers.ts b/.github/actions/codex/src/git-helpers.ts new file mode 100644 index 0000000000..001ccde354 --- /dev/null +++ b/.github/actions/codex/src/git-helpers.ts @@ -0,0 +1,149 @@ +import { spawnSync } from "child_process"; +import * as github from "@actions/github"; +import { EnvContext } from "./env-context"; + +function runGit(args: string[], silent = true): string { + console.info(`Running git ${args.join(" ")}`); + const res = spawnSync("git", args, { + encoding: "utf8", + stdio: silent ? ["ignore", "pipe", "pipe"] : "inherit", + }); + if (res.error) { + throw res.error; + } + if (res.status !== 0) { + // Return stderr so caller may handle; else throw. + throw new Error( + `git ${args.join(" ")} failed with code ${res.status}: ${res.stderr}`, + ); + } + return res.stdout.trim(); +} + +function stageAllChanges() { + runGit(["add", "-A"]); +} + +function hasStagedChanges(): boolean { + const res = spawnSync("git", ["diff", "--cached", "--quiet", "--exit-code"]); + return res.status !== 0; +} + +function ensureOnBranch( + issueNumber: number, + protectedBranches: string[], + suggestedSlug?: string, +): string { + let branch = ""; + try { + branch = runGit(["symbolic-ref", "--short", "-q", "HEAD"]); + } catch { + branch = ""; + } + + // If detached HEAD or on a protected branch, create a new branch. + if (!branch || protectedBranches.includes(branch)) { + if (suggestedSlug) { + const safeSlug = suggestedSlug + .toLowerCase() + .replace(/[^\w\s-]/g, "") + .trim() + .replace(/\s+/g, "-"); + branch = `codex-fix-${issueNumber}-${safeSlug}`; + } else { + branch = `codex-fix-${issueNumber}-${Date.now()}`; + } + runGit(["switch", "-c", branch]); + } + return branch; +} + +function commitIfNeeded(issueNumber: number) { + if (hasStagedChanges()) { + runGit([ + "commit", + "-m", + `fix: automated fix for #${issueNumber} via Codex`, + ]); + } +} + +function pushBranch(branch: string, githubToken: string, ctx: EnvContext) { + const repoSlug = ctx.get("GITHUB_REPOSITORY"); // owner/repo + const remoteUrl = `https://x-access-token:${githubToken}@github.com/${repoSlug}.git`; + + runGit(["push", "--force-with-lease", "-u", remoteUrl, `HEAD:${branch}`]); +} + +/** + * If this returns a string, it is the URL of the created PR. + */ +export async function maybePublishPRForIssue( + issueNumber: number, + lastMessage: string, + ctx: EnvContext, +): Promise { + // Only proceed if GITHUB_TOKEN available. + const githubToken = + ctx.tryGetNonEmpty("GITHUB_TOKEN") ?? ctx.tryGetNonEmpty("GH_TOKEN"); + if (!githubToken) { + console.warn("No GitHub token - skipping PR creation."); + return undefined; + } + + // Print `git status` for debugging. + runGit(["status"]); + + // Stage any remaining changes so they can be committed and pushed. + stageAllChanges(); + + const octokit = ctx.getOctokit(githubToken); + + const { owner, repo } = github.context.repo; + + // Determine default branch to treat as protected. + let defaultBranch = "main"; + try { + const repoInfo = await octokit.rest.repos.get({ owner, repo }); + defaultBranch = repoInfo.data.default_branch ?? "main"; + } catch (e) { + console.warn(`Failed to get default branch, assuming 'main': ${e}`); + } + + const sanitizedMessage = lastMessage.replace(/\u2022/g, "-"); + const [summaryLine] = sanitizedMessage.split(/\r?\n/); + const branch = ensureOnBranch(issueNumber, [defaultBranch, "master"], summaryLine); + commitIfNeeded(issueNumber); + pushBranch(branch, githubToken, ctx); + + // Try to find existing PR for this branch + const headParam = `${owner}:${branch}`; + const existing = await octokit.rest.pulls.list({ + owner, + repo, + head: headParam, + state: "open", + }); + if (existing.data.length > 0) { + return existing.data[0].html_url; + } + + // Determine base branch (default to main) + let baseBranch = "main"; + try { + const repoInfo = await octokit.rest.repos.get({ owner, repo }); + baseBranch = repoInfo.data.default_branch ?? "main"; + } catch (e) { + console.warn(`Failed to get default branch, assuming 'main': ${e}`); + } + + const pr = await octokit.rest.pulls.create({ + owner, + repo, + title: summaryLine, + head: branch, + base: baseBranch, + body: sanitizedMessage, + }); + return pr.data.html_url; +} diff --git a/.github/actions/codex/src/git-user.ts b/.github/actions/codex/src/git-user.ts new file mode 100644 index 0000000000..bd84a61a7b --- /dev/null +++ b/.github/actions/codex/src/git-user.ts @@ -0,0 +1,16 @@ +export function setGitHubActionsUser(): void { + const commands = [ + ["git", "config", "--global", "user.name", "github-actions[bot]"], + [ + "git", + "config", + "--global", + "user.email", + "41898282+github-actions[bot]@users.noreply.github.com", + ], + ]; + + for (const command of commands) { + Bun.spawnSync(command); + } +} diff --git a/.github/actions/codex/src/github-workspace.ts b/.github/actions/codex/src/github-workspace.ts new file mode 100644 index 0000000000..8a1f7cae50 --- /dev/null +++ b/.github/actions/codex/src/github-workspace.ts @@ -0,0 +1,11 @@ +import * as pathMod from "path"; +import { EnvContext } from "./env-context"; + +export function resolveWorkspacePath(path: string, ctx: EnvContext): string { + if (pathMod.isAbsolute(path)) { + return path; + } else { + const workspace = ctx.get("GITHUB_WORKSPACE"); + return pathMod.join(workspace, path); + } +} diff --git a/.github/actions/codex/src/load-config.ts b/.github/actions/codex/src/load-config.ts new file mode 100644 index 0000000000..f225e81a0c --- /dev/null +++ b/.github/actions/codex/src/load-config.ts @@ -0,0 +1,56 @@ +import type { Config, LabelConfig } from "./config"; + +import { getDefaultConfig } from "./default-label-config"; +import { readFileSync, readdirSync, statSync } from "fs"; +import * as path from "path"; + +/** + * Build an in-memory configuration object by scanning the repository for + * Markdown templates located in `.github/codex/labels`. + * + * Each `*.md` file in that directory represents a label that can trigger the + * Codex GitHub Action. The filename **without** the extension is interpreted + * as the label name, e.g. `codex-review.md` ➜ `codex-review`. + * + * For every such label we derive the corresponding `doneLabel` by appending + * the suffix `-completed`. + */ +export function loadConfig(workspace: string): Config { + const labelsDir = path.join(workspace, ".github", "codex", "labels"); + + let entries: string[]; + try { + entries = readdirSync(labelsDir); + } catch { + // If the directory is missing, return the default configuration. + return getDefaultConfig(); + } + + const labels: Record = {}; + + for (const entry of entries) { + if (!entry.endsWith(".md")) { + continue; + } + + const fullPath = path.join(labelsDir, entry); + + if (!statSync(fullPath).isFile()) { + continue; + } + + const labelName = entry.slice(0, -3); // trim ".md" + + labels[labelName] = new FileLabelConfig(fullPath); + } + + return { labels }; +} + +class FileLabelConfig implements LabelConfig { + constructor(private readonly promptPath: string) {} + + getPromptTemplate(): string { + return readFileSync(this.promptPath, "utf8"); + } +} diff --git a/.github/actions/codex/src/main.ts b/.github/actions/codex/src/main.ts new file mode 100755 index 0000000000..a334c68917 --- /dev/null +++ b/.github/actions/codex/src/main.ts @@ -0,0 +1,80 @@ +#!/usr/bin/env bun + +import type { Config } from "./config"; + +import { defaultContext, EnvContext } from "./env-context"; +import { loadConfig } from "./load-config"; +import { setGitHubActionsUser } from "./git-user"; +import { onLabeled } from "./process-label"; +import { ensureBaseAndHeadCommitsForPRAreAvailable } from "./prompt-template"; +import { performAdditionalValidation } from "./verify-inputs"; +import { onComment } from "./comment"; +import { onReview } from "./review"; + +async function main(): Promise { + const ctx: EnvContext = defaultContext; + + // Build the configuration dynamically by scanning `.github/codex/labels`. + const GITHUB_WORKSPACE = ctx.get("GITHUB_WORKSPACE"); + const config: Config = loadConfig(GITHUB_WORKSPACE); + + // Optionally perform additional validation of prompt template files. + performAdditionalValidation(config, GITHUB_WORKSPACE); + + const GITHUB_EVENT_NAME = ctx.get("GITHUB_EVENT_NAME"); + const GITHUB_EVENT_ACTION = ctx.get("GITHUB_EVENT_ACTION"); + + // Set user.name and user.email to a bot before Codex runs, just in case it + // creates a commit. + setGitHubActionsUser(); + + switch (GITHUB_EVENT_NAME) { + case "issues": { + if (GITHUB_EVENT_ACTION === "labeled") { + await onLabeled(config, ctx); + return; + } else if (GITHUB_EVENT_ACTION === "opened") { + await onComment(ctx); + return; + } + break; + } + case "issue_comment": { + if (GITHUB_EVENT_ACTION === "created") { + await onComment(ctx); + return; + } + break; + } + case "pull_request": { + if (GITHUB_EVENT_ACTION === "labeled") { + await ensureBaseAndHeadCommitsForPRAreAvailable(ctx); + await onLabeled(config, ctx); + return; + } + break; + } + case "pull_request_review": { + await ensureBaseAndHeadCommitsForPRAreAvailable(ctx); + if (GITHUB_EVENT_ACTION === "submitted") { + await onReview(ctx); + return; + } + break; + } + case "pull_request_review_comment": { + await ensureBaseAndHeadCommitsForPRAreAvailable(ctx); + if (GITHUB_EVENT_ACTION === "created") { + await onComment(ctx); + return; + } + break; + } + } + + console.warn( + `Unsupported action '${GITHUB_EVENT_ACTION}' for event '${GITHUB_EVENT_NAME}'.`, + ); +} + +main(); diff --git a/.github/actions/codex/src/post-comment.ts b/.github/actions/codex/src/post-comment.ts new file mode 100644 index 0000000000..9a3d7528eb --- /dev/null +++ b/.github/actions/codex/src/post-comment.ts @@ -0,0 +1,60 @@ +import { fail } from "./fail"; +import * as github from "@actions/github"; +import { EnvContext } from "./env-context"; + +/** + * Post a comment to the issue / pull request currently in scope. + * + * Provide the environment context so that token lookup (inside getOctokit) does + * not rely on global state. + */ +export async function postComment( + commentBody: string, + ctx: EnvContext, +): Promise { + // Append a footer with a link back to the workflow run, if available. + const footer = buildWorkflowRunFooter(ctx); + const bodyWithFooter = footer ? `${commentBody}${footer}` : commentBody; + + const octokit = ctx.getOctokit(); + const { owner, repo } = github.context.repo; + const issueNumber = github.context.issue.number; + + if (!issueNumber) { + console.warn( + "No issue or pull_request number found in GitHub context; skipping comment creation.", + ); + return; + } + + try { + await octokit.rest.issues.createComment({ + owner, + repo, + issue_number: issueNumber, + body: bodyWithFooter, + }); + } catch (error) { + fail(`Failed to create comment via GitHub API: ${error}`); + } +} + +/** + * Helper to build a Markdown fragment linking back to the workflow run that + * generated the current comment. Returns `undefined` if required environment + * variables are missing – e.g. when running outside of GitHub Actions – so we + * can gracefully skip the footer in those cases. + */ +function buildWorkflowRunFooter(ctx: EnvContext): string | undefined { + const serverUrl = + ctx.tryGetNonEmpty("GITHUB_SERVER_URL") ?? "https://github.com"; + const repository = ctx.tryGetNonEmpty("GITHUB_REPOSITORY"); + const runId = ctx.tryGetNonEmpty("GITHUB_RUN_ID"); + + if (!repository || !runId) { + return undefined; + } + + const url = `${serverUrl}/${repository}/actions/runs/${runId}`; + return `\n\n---\n*[_View workflow run_](${url})*`; +} diff --git a/.github/actions/codex/src/process-label.ts b/.github/actions/codex/src/process-label.ts new file mode 100644 index 0000000000..4b4361e118 --- /dev/null +++ b/.github/actions/codex/src/process-label.ts @@ -0,0 +1,195 @@ +import { fail } from "./fail"; +import { EnvContext } from "./env-context"; +import { renderPromptTemplate } from "./prompt-template"; + +import { postComment } from "./post-comment"; +import { runCodex } from "./run-codex"; + +import * as github from "@actions/github"; +import { Config, LabelConfig } from "./config"; +import { maybePublishPRForIssue } from "./git-helpers"; + +export async function onLabeled( + config: Config, + ctx: EnvContext, +): Promise { + const GITHUB_EVENT_LABEL_NAME = ctx.get("GITHUB_EVENT_LABEL_NAME"); + const labelConfig = config.labels[GITHUB_EVENT_LABEL_NAME] as + | LabelConfig + | undefined; + if (!labelConfig) { + fail( + `Label \`${GITHUB_EVENT_LABEL_NAME}\` not found in config: ${JSON.stringify(config)}`, + ); + } + + await processLabelConfig(ctx, GITHUB_EVENT_LABEL_NAME, labelConfig); +} + +/** + * Wrapper that handles `-in-progress` and `-completed` semantics around the core lint/fix/review + * processing. It will: + * + * - Skip execution if the `-in-progress` or `-completed` label is already present. + * - Mark the PR/issue as `-in-progress`. + * - After successful execution, mark the PR/issue as `-completed`. + */ +async function processLabelConfig( + ctx: EnvContext, + label: string, + labelConfig: LabelConfig, +): Promise { + const octokit = ctx.getOctokit(); + const { owner, repo, issueNumber, labelNames } = + await getCurrentLabels(octokit); + + const inProgressLabel = `${label}-in-progress`; + const completedLabel = `${label}-completed`; + for (const markerLabel of [inProgressLabel, completedLabel]) { + if (labelNames.includes(markerLabel)) { + console.log( + `Label '${markerLabel}' already present on issue/PR #${issueNumber}. Skipping Codex action.`, + ); + + // Clean up: remove the triggering label to avoid confusion and re-runs. + await addAndRemoveLabels(octokit, { + owner, + repo, + issueNumber, + remove: markerLabel, + }); + + return; + } + } + + // Mark the PR/issue as in progress. + await addAndRemoveLabels(octokit, { + owner, + repo, + issueNumber, + add: inProgressLabel, + remove: label, + }); + + // Run the core Codex processing. + await processLabel(ctx, label, labelConfig); + + // Mark the PR/issue as completed. + await addAndRemoveLabels(octokit, { + owner, + repo, + issueNumber, + add: completedLabel, + remove: inProgressLabel, + }); +} + +async function processLabel( + ctx: EnvContext, + label: string, + labelConfig: LabelConfig, +): Promise { + const template = labelConfig.getPromptTemplate(); + const populatedTemplate = await renderPromptTemplate(template, ctx); + + // Always run Codex and post the resulting message as a comment. + let commentBody = await runCodex(populatedTemplate, ctx); + + // Current heuristic: only try to create a PR if "attempt" or "fix" is in the + // label name. (Yes, we plan to evolve this.) + if (label.indexOf("fix") !== -1 || label.indexOf("attempt") !== -1) { + console.info(`label ${label} indicates we should attempt to create a PR`); + const prUrl = await maybeFixIssue(ctx, commentBody); + if (prUrl) { + commentBody += `\n\n---\nOpened pull request: ${prUrl}`; + } + } else { + console.info( + `label ${label} does not indicate we should attempt to create a PR`, + ); + } + + await postComment(commentBody, ctx); +} + +async function maybeFixIssue( + ctx: EnvContext, + lastMessage: string, +): Promise { + // Attempt to create a PR out of any changes Codex produced. + const issueNumber = github.context.issue.number!; // exists for issues triggering this path + try { + return await maybePublishPRForIssue(issueNumber, lastMessage, ctx); + } catch (e) { + console.warn(`Failed to publish PR: ${e}`); + } +} + +async function getCurrentLabels( + octokit: ReturnType, +): Promise<{ + owner: string; + repo: string; + issueNumber: number; + labelNames: Array; +}> { + const { owner, repo } = github.context.repo; + const issueNumber = github.context.issue.number; + + if (!issueNumber) { + fail("No issue or pull_request number found in GitHub context."); + } + + const { data: issueData } = await octokit.rest.issues.get({ + owner, + repo, + issue_number: issueNumber, + }); + + const labelNames = + issueData.labels?.map((label: any) => + typeof label === "string" ? label : label.name, + ) ?? []; + + return { owner, repo, issueNumber, labelNames }; +} + +async function addAndRemoveLabels( + octokit: ReturnType, + opts: { + owner: string; + repo: string; + issueNumber: number; + add?: string; + remove?: string; + }, +): Promise { + const { owner, repo, issueNumber, add, remove } = opts; + + if (add) { + try { + await octokit.rest.issues.addLabels({ + owner, + repo, + issue_number: issueNumber, + labels: [add], + }); + } catch (error) { + console.warn(`Failed to add label '${add}': ${error}`); + } + } + + if (remove) { + try { + await octokit.rest.issues.removeLabel({ + owner, + repo, + issue_number: issueNumber, + name: remove, + }); + } catch (error) { + console.warn(`Failed to remove label '${remove}': ${error}`); + } + } +} diff --git a/.github/actions/codex/src/prompt-template.ts b/.github/actions/codex/src/prompt-template.ts new file mode 100644 index 0000000000..aa52dd2af2 --- /dev/null +++ b/.github/actions/codex/src/prompt-template.ts @@ -0,0 +1,284 @@ +/* + * Utilities to render Codex prompt templates. + * + * A template is a Markdown (or plain-text) file that may contain one or more + * placeholders of the form `{CODEX_ACTION_}`. At runtime these + * placeholders are substituted with dynamically generated content. Each + * placeholder is resolved **exactly once** even if it appears multiple times + * in the same template. + */ + +import { readFile } from "fs/promises"; + +import { EnvContext } from "./env-context"; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +/** + * Lazily caches parsed `$GITHUB_EVENT_PATH` contents keyed by the file path so + * we only hit the filesystem once per unique event payload. + */ +const githubEventDataCache: Map> = new Map(); + +function getGitHubEventData(ctx: EnvContext): Promise { + const eventPath = ctx.get("GITHUB_EVENT_PATH"); + let cached = githubEventDataCache.get(eventPath); + if (!cached) { + cached = readFile(eventPath, "utf8").then((raw) => JSON.parse(raw)); + githubEventDataCache.set(eventPath, cached); + } + return cached; +} + +async function runCommand(args: Array): Promise { + const result = Bun.spawnSync(args, { + stdout: "pipe", + stderr: "pipe", + }); + + if (result.success) { + return result.stdout.toString(); + } + + console.error(`Error running ${JSON.stringify(args)}: ${result.stderr}`); + return ""; +} + +// --------------------------------------------------------------------------- +// Public API +// --------------------------------------------------------------------------- + +// Regex that captures the variable name without the surrounding { } braces. +const VAR_REGEX = /\{(CODEX_ACTION_[A-Z0-9_]+)\}/g; + +// Cache individual placeholder values so each one is resolved at most once per +// process even if many templates reference it. +const placeholderCache: Map> = new Map(); + +/** + * Parse a template string, resolve all placeholders and return the rendered + * result. + */ +export async function renderPromptTemplate( + template: string, + ctx: EnvContext, +): Promise { + // --------------------------------------------------------------------- + // 1) Gather all *unique* placeholders present in the template. + // --------------------------------------------------------------------- + const variables = new Set(); + for (const match of template.matchAll(VAR_REGEX)) { + variables.add(match[1]); + } + + // --------------------------------------------------------------------- + // 2) Kick off (or reuse) async resolution for each variable. + // --------------------------------------------------------------------- + for (const variable of variables) { + if (!placeholderCache.has(variable)) { + placeholderCache.set(variable, resolveVariable(variable, ctx)); + } + } + + // --------------------------------------------------------------------- + // 3) Await completion so we can perform a simple synchronous replace below. + // --------------------------------------------------------------------- + const resolvedEntries: [string, string][] = []; + for (const [key, promise] of placeholderCache.entries()) { + resolvedEntries.push([key, await promise]); + } + const resolvedMap = new Map(resolvedEntries); + + // --------------------------------------------------------------------- + // 4) Replace each occurrence. We use replace with a callback to ensure + // correct substitution even if variable names overlap (they shouldn't, + // but better safe than sorry). + // --------------------------------------------------------------------- + return template.replace(VAR_REGEX, (_, varName: string) => { + return resolvedMap.get(varName) ?? ""; + }); +} + +export async function ensureBaseAndHeadCommitsForPRAreAvailable( + ctx: EnvContext, +): Promise<{ baseSha: string; headSha: string } | null> { + const prShas = await getPrShas(ctx); + if (prShas == null) { + console.warn("Unable to resolve PR branches"); + return null; + } + + const event = await getGitHubEventData(ctx); + const pr = event.pull_request; + if (!pr) { + console.warn("event.pull_request is not defined - unexpected"); + return null; + } + + const workspace = ctx.get("GITHUB_WORKSPACE"); + + // Refs (branch names) + const baseRef: string | undefined = pr.base?.ref; + const headRef: string | undefined = pr.head?.ref; + + // Clone URLs + const baseRemoteUrl: string | undefined = pr.base?.repo?.clone_url; + const headRemoteUrl: string | undefined = pr.head?.repo?.clone_url; + + if (!baseRef || !headRef || !baseRemoteUrl || !headRemoteUrl) { + console.warn( + "Missing PR ref or remote URL information - cannot fetch commits", + ); + return null; + } + + // Ensure we have the base branch. + await runCommand([ + "git", + "-C", + workspace, + "fetch", + "--no-tags", + "origin", + baseRef, + ]); + + // Ensure we have the head branch. + if (headRemoteUrl === baseRemoteUrl) { + // Same repository – the commit is available from `origin`. + await runCommand([ + "git", + "-C", + workspace, + "fetch", + "--no-tags", + "origin", + headRef, + ]); + } else { + // Fork – make sure a `pr` remote exists that points at the fork. Attempting + // to add a remote that already exists causes git to error, so we swallow + // any non-zero exit codes from that specific command. + await runCommand([ + "git", + "-C", + workspace, + "remote", + "add", + "pr", + headRemoteUrl, + ]); + + // Whether adding succeeded or the remote already existed, attempt to fetch + // the head ref from the `pr` remote. + await runCommand([ + "git", + "-C", + workspace, + "fetch", + "--no-tags", + "pr", + headRef, + ]); + } + + return prShas; +} + +// --------------------------------------------------------------------------- +// Internal helpers – still exported for use by other modules. +// --------------------------------------------------------------------------- + +export async function resolvePrDiff(ctx: EnvContext): Promise { + const prShas = await ensureBaseAndHeadCommitsForPRAreAvailable(ctx); + if (prShas == null) { + console.warn("Unable to resolve PR branches"); + return ""; + } + + const workspace = ctx.get("GITHUB_WORKSPACE"); + const { baseSha, headSha } = prShas; + return runCommand([ + "git", + "-C", + workspace, + "diff", + "--color=never", + `${baseSha}..${headSha}`, + ]); +} + +// --------------------------------------------------------------------------- +// Placeholder resolution +// --------------------------------------------------------------------------- + +async function resolveVariable(name: string, ctx: EnvContext): Promise { + switch (name) { + case "CODEX_ACTION_ISSUE_TITLE": { + const event = await getGitHubEventData(ctx); + const issue = event.issue ?? event.pull_request; + return issue?.title ?? ""; + } + + case "CODEX_ACTION_ISSUE_BODY": { + const event = await getGitHubEventData(ctx); + const issue = event.issue ?? event.pull_request; + return issue?.body ?? ""; + } + + case "CODEX_ACTION_GITHUB_EVENT_PATH": { + return ctx.get("GITHUB_EVENT_PATH"); + } + + case "CODEX_ACTION_BASE_REF": { + const event = await getGitHubEventData(ctx); + return event?.pull_request?.base?.ref ?? ""; + } + + case "CODEX_ACTION_HEAD_REF": { + const event = await getGitHubEventData(ctx); + return event?.pull_request?.head?.ref ?? ""; + } + + case "CODEX_ACTION_PR_DIFF": { + return resolvePrDiff(ctx); + } + + // ------------------------------------------------------------------- + // Add new template variables here. + // ------------------------------------------------------------------- + + default: { + // Unknown variable – leave it blank to avoid leaking placeholders to the + // final prompt. The alternative would be to `fail()` here, but silently + // ignoring unknown placeholders is more forgiving and better matches the + // behaviour of typical template engines. + console.warn(`Unknown template variable: ${name}`); + return ""; + } + } +} + +async function getPrShas( + ctx: EnvContext, +): Promise<{ baseSha: string; headSha: string } | null> { + const event = await getGitHubEventData(ctx); + const pr = event.pull_request; + if (!pr) { + console.warn("event.pull_request is not defined"); + return null; + } + + // Prefer explicit SHAs if available to avoid relying on local branch names. + const baseSha: string | undefined = pr.base?.sha; + const headSha: string | undefined = pr.head?.sha; + + if (!baseSha || !headSha) { + console.warn("one of base or head is not defined on event.pull_request"); + return null; + } + + return { baseSha, headSha }; +} diff --git a/.github/actions/codex/src/review.ts b/.github/actions/codex/src/review.ts new file mode 100644 index 0000000000..64f826dcc5 --- /dev/null +++ b/.github/actions/codex/src/review.ts @@ -0,0 +1,42 @@ +import type { EnvContext } from "./env-context"; +import { runCodex } from "./run-codex"; +import { postComment } from "./post-comment"; +import { addEyesReaction } from "./add-reaction"; + +/** + * Handle `pull_request_review` events. We treat the review body the same way + * as a normal comment. + */ +export async function onReview(ctx: EnvContext): Promise { + const triggerPhrase = ctx.tryGet("INPUT_TRIGGER_PHRASE"); + if (!triggerPhrase) { + console.warn("Empty trigger phrase: skipping."); + return; + } + + const reviewBody = ctx.tryGet("GITHUB_EVENT_REVIEW_BODY"); + + if (!reviewBody) { + console.warn("Review body not found in environment: skipping."); + return; + } + + if (!reviewBody.includes(triggerPhrase)) { + console.log( + `Trigger phrase '${triggerPhrase}' not found: nothing to do for this review.`, + ); + return; + } + + const prompt = reviewBody.replace(triggerPhrase, "").trim(); + + if (prompt.length === 0) { + console.warn("Prompt is empty after removing trigger phrase: skipping."); + return; + } + + await addEyesReaction(ctx); + + const lastMessage = await runCodex(prompt, ctx); + await postComment(lastMessage, ctx); +} diff --git a/.github/actions/codex/src/run-codex.ts b/.github/actions/codex/src/run-codex.ts new file mode 100644 index 0000000000..2c851823e8 --- /dev/null +++ b/.github/actions/codex/src/run-codex.ts @@ -0,0 +1,56 @@ +import { fail } from "./fail"; +import { EnvContext } from "./env-context"; +import { tmpdir } from "os"; +import { join } from "node:path"; +import { readFile, mkdtemp } from "fs/promises"; +import { resolveWorkspacePath } from "./github-workspace"; + +/** + * Runs the Codex CLI with the provided prompt and returns the output written + * to the "last message" file. + */ +export async function runCodex( + prompt: string, + ctx: EnvContext, +): Promise { + const OPENAI_API_KEY = ctx.get("OPENAI_API_KEY"); + + const tempDirPath = await mkdtemp(join(tmpdir(), "codex-")); + const lastMessageOutput = join(tempDirPath, "codex-prompt.md"); + + const args = ["/usr/local/bin/codex-exec"]; + + const inputCodexArgs = ctx.tryGet("INPUT_CODEX_ARGS")?.trim(); + if (inputCodexArgs) { + args.push(...inputCodexArgs.split(/\s+/)); + } + + args.push("--output-last-message", lastMessageOutput, prompt); + + const env: Record = { ...process.env, OPENAI_API_KEY }; + const INPUT_CODEX_HOME = ctx.tryGet("INPUT_CODEX_HOME"); + if (INPUT_CODEX_HOME) { + env.CODEX_HOME = resolveWorkspacePath(INPUT_CODEX_HOME, ctx); + } + + console.log(`Running Codex: ${JSON.stringify(args)}`); + const result = Bun.spawnSync(args, { + stdout: "inherit", + stderr: "inherit", + env, + }); + + if (!result.success) { + fail(`Codex failed: see above for details.`); + } + + // Read the output generated by Codex. + let lastMessage: string; + try { + lastMessage = await readFile(lastMessageOutput, "utf8"); + } catch (err) { + fail(`Failed to read Codex output at '${lastMessageOutput}': ${err}`); + } + + return lastMessage; +} diff --git a/.github/actions/codex/src/verify-inputs.ts b/.github/actions/codex/src/verify-inputs.ts new file mode 100644 index 0000000000..bfc5dcda83 --- /dev/null +++ b/.github/actions/codex/src/verify-inputs.ts @@ -0,0 +1,33 @@ +// Validate the inputs passed to the composite action. +// The script currently ensures that the provided configuration file exists and +// matches the expected schema. + +import type { Config } from "./config"; + +import { existsSync } from "fs"; +import * as path from "path"; +import { fail } from "./fail"; + +export function performAdditionalValidation(config: Config, workspace: string) { + // Additional validation: ensure referenced prompt files exist and are Markdown. + for (const [label, details] of Object.entries(config.labels)) { + // Determine which prompt key is present (the schema guarantees exactly one). + const promptPathStr = + (details as any).prompt ?? (details as any).promptPath; + + if (promptPathStr) { + const promptPath = path.isAbsolute(promptPathStr) + ? promptPathStr + : path.join(workspace, promptPathStr); + + if (!existsSync(promptPath)) { + fail(`Prompt file for label '${label}' not found: ${promptPath}`); + } + if (!promptPath.endsWith(".md")) { + fail( + `Prompt file for label '${label}' must be a .md file (got ${promptPathStr}).`, + ); + } + } + } +} diff --git a/.github/actions/codex/tsconfig.json b/.github/actions/codex/tsconfig.json new file mode 100644 index 0000000000..c05c2955bf --- /dev/null +++ b/.github/actions/codex/tsconfig.json @@ -0,0 +1,15 @@ +{ + "compilerOptions": { + "lib": ["ESNext"], + "target": "ESNext", + "module": "ESNext", + "moduleDetection": "force", + "moduleResolution": "bundler", + + "noEmit": true, + "strict": true, + "skipLibCheck": true + }, + + "include": ["src"] +} diff --git a/.github/codex/home/config.toml b/.github/codex/home/config.toml new file mode 100644 index 0000000000..bb1b362bb6 --- /dev/null +++ b/.github/codex/home/config.toml @@ -0,0 +1,3 @@ +model = "o3" + +# Consider setting [mcp_servers] here! diff --git a/.github/codex/labels/codex-attempt.md b/.github/codex/labels/codex-attempt.md new file mode 100644 index 0000000000..b2a3e93af2 --- /dev/null +++ b/.github/codex/labels/codex-attempt.md @@ -0,0 +1,9 @@ +Attempt to solve the reported issue. + +If a code change is required, create a new branch, commit the fix, and open a pull request that resolves the problem. + +Here is the original GitHub issue that triggered this run: + +### {CODEX_ACTION_ISSUE_TITLE} + +{CODEX_ACTION_ISSUE_BODY} diff --git a/.github/codex/labels/codex-review.md b/.github/codex/labels/codex-review.md new file mode 100644 index 0000000000..7c6c14ad57 --- /dev/null +++ b/.github/codex/labels/codex-review.md @@ -0,0 +1,7 @@ +Review this PR and respond with a very concise final message, formatted in Markdown. + +There should be a summary of the changes (1-2 sentences) and a few bullet points if necessary. + +Then provide the **review** (1-2 sentences plus bullet points, friendly tone). + +{CODEX_ACTION_GITHUB_EVENT_PATH} contains the JSON that triggered this GitHub workflow. It contains the `base` and `head` refs that define this PR. Both refs are available locally. diff --git a/.github/codex/labels/codex-triage.md b/.github/codex/labels/codex-triage.md new file mode 100644 index 0000000000..46ed362416 --- /dev/null +++ b/.github/codex/labels/codex-triage.md @@ -0,0 +1,7 @@ +Troubleshoot whether the reported issue is valid. + +Provide a concise and respectful comment summarizing the findings. + +### {CODEX_ACTION_ISSUE_TITLE} + +{CODEX_ACTION_ISSUE_BODY} diff --git a/.github/workflows/codex.yml b/.github/workflows/codex.yml new file mode 100644 index 0000000000..0df24c8a79 --- /dev/null +++ b/.github/workflows/codex.yml @@ -0,0 +1,76 @@ +name: Codex + +on: + issues: + types: [opened, labeled] + pull_request: + branches: [main] + types: [labeled] + +jobs: + codex: + # This `if` check provides complex filtering logic to avoid running Codex + # on every PR. Admittedly, one thing this does not verify is whether the + # sender has write access to the repo: that must be done as part of a + # runtime step. + # + # Note the label values should match the ones in the .github/codex/labels + # folder. + if: | + (github.event_name == 'issues' && ( + (github.event.action == 'labeled' && (github.event.label.name == 'codex-attempt' || github.event.label.name == 'codex-triage')) + )) || + (github.event_name == 'pull_request' && github.event.action == 'labeled' && github.event.label.name == 'codex-review') + runs-on: ubuntu-latest + permissions: + contents: write # can push or create branches + issues: write # for comments + labels on issues/PRs + pull-requests: write # for PR comments/labels + steps: + # TODO: Consider adding an optional mode (--dry-run?) to actions/codex + # that verifies whether Codex should actually be run for this event. + # (For example, it may be rejected because the sender does not have + # write access to the repo.) The benefit would be two-fold: + # 1. As the first step of this job, it gives us a chance to add a reaction + # or comment to the PR/issue ASAP to "ack" the request. + # 2. It saves resources by skipping the clone and setup steps below if + # Codex is not going to run. + + - name: Checkout repository + uses: actions/checkout@v4 + + # We install the dependencies like we would for an ordinary CI job, + # particularly because Codex will not have network access to install + # these dependencies. + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: 22 + + - name: Install dependencies (codex-cli) + working-directory: codex-cli + run: npm ci + + - uses: dtolnay/rust-toolchain@1.87 + with: + targets: x86_64-unknown-linux-gnu + components: clippy + + - uses: actions/cache@v4 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + ${{ github.workspace }}/codex-rs/target/ + key: cargo-ubuntu-24.04-x86_64-unknown-linux-gnu-${{ hashFiles('**/Cargo.lock') }} + + # Note it is possible that the `verify` step internal to Run Codex will + # fail, in which case the work to setup the repo was worthless :( + - name: Run Codex + uses: ./.github/actions/codex + with: + openai_api_key: ${{ secrets.CODEX_OPENAI_API_KEY }} + github_token: ${{ secrets.GITHUB_TOKEN }} + codex_home: ./.github/codex/home From 867618538906a36582d4e7315c0c1456ee979999 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Fri, 30 May 2025 10:58:57 -0700 Subject: [PATCH 13/41] fix: update outdated repo setup in codex.yml (#1171) We should do some work to share the setup logic across `codex.yml`, `ci.yml`, and `rust-ci.yml`. --- .github/workflows/codex.yml | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codex.yml b/.github/workflows/codex.yml index 0df24c8a79..1ca50f499f 100644 --- a/.github/workflows/codex.yml +++ b/.github/workflows/codex.yml @@ -47,9 +47,22 @@ jobs: with: node-version: 22 - - name: Install dependencies (codex-cli) - working-directory: codex-cli - run: npm ci + - name: Get pnpm store directory + id: pnpm-cache + shell: bash + run: | + echo "store_path=$(pnpm store path --silent)" >> $GITHUB_OUTPUT + + - name: Setup pnpm cache + uses: actions/cache@v4 + with: + path: ${{ steps.pnpm-cache.outputs.store_path }} + key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }} + restore-keys: | + ${{ runner.os }}-pnpm-store- + + - name: Install dependencies + run: pnpm install - uses: dtolnay/rust-toolchain@1.87 with: From 0f40ef5a10aa3b9a04f9105668b4bf7c69c3ae83 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Fri, 30 May 2025 11:04:41 -0700 Subject: [PATCH 14/41] fix: missed a step in #1171 for codex.yml (#1172) Missed in my copy/paste. --- .github/workflows/codex.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/codex.yml b/.github/workflows/codex.yml index 1ca50f499f..a105581bbc 100644 --- a/.github/workflows/codex.yml +++ b/.github/workflows/codex.yml @@ -47,6 +47,12 @@ jobs: with: node-version: 22 + - name: Setup pnpm + uses: pnpm/action-setup@v4 + with: + version: 10.8.1 + run_install: false + - name: Get pnpm store directory id: pnpm-cache shell: bash From e207f20f6419b9d7d1ff3f6364e6295c6d5a87ae Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Fri, 30 May 2025 11:16:30 -0700 Subject: [PATCH 15/41] fix: add extra debugging to GitHub Action (#1173) https://github.com/openai/codex/actions/runs/15352839832/job/43205041563 appeared to fail around `postComment()`, but I don't see the output from `fail()` in the logs. Adding a bit more info. --- .github/actions/codex/src/post-comment.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/actions/codex/src/post-comment.ts b/.github/actions/codex/src/post-comment.ts index 9a3d7528eb..914fd0d322 100644 --- a/.github/actions/codex/src/post-comment.ts +++ b/.github/actions/codex/src/post-comment.ts @@ -17,6 +17,7 @@ export async function postComment( const bodyWithFooter = footer ? `${commentBody}${footer}` : commentBody; const octokit = ctx.getOctokit(); + console.info("Got Octokit instance for posting comment"); const { owner, repo } = github.context.repo; const issueNumber = github.context.issue.number; @@ -28,6 +29,7 @@ export async function postComment( } try { + console.info("Calling octokit.rest.issues.createComment()"); await octokit.rest.issues.createComment({ owner, repo, From 1bf82056b35a80e271e7791a1c8421e69566f33f Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Fri, 30 May 2025 14:07:03 -0700 Subject: [PATCH 16/41] fix: introduce `create_tools_json()` and share it with chat_completions.rs (#1177) The main motivator behind this PR is that `stream_chat_completions()` was not adding the `"tools"` entry to the payload posted to the `/chat/completions` endpoint. This (1) refactors the existing logic to build up the `"tools"` JSON from `client.rs` into `openai_tools.rs`, and (2) updates the use of responses API (`client.rs`) and chat completions API (`chat_completions.rs`) to both use it. Note this PR alone is not sufficient to get tool calling from chat completions working: that is done in https://github.com/openai/codex/pull/1167. --- [//]: # (BEGIN SAPLING FOOTER) Stack created with [Sapling](https://sapling-scm.com). Best reviewed with [ReviewStack](https://reviewstack.dev/openai/codex/pull/1177). * #1167 * __->__ #1177 --- codex-rs/core/src/chat_completions.rs | 10 +- codex-rs/core/src/client.rs | 116 +------------------ codex-rs/core/src/lib.rs | 1 + codex-rs/core/src/openai_tools.rs | 158 ++++++++++++++++++++++++++ 4 files changed, 169 insertions(+), 116 deletions(-) create mode 100644 codex-rs/core/src/openai_tools.rs diff --git a/codex-rs/core/src/chat_completions.rs b/codex-rs/core/src/chat_completions.rs index 7760c48fbf..f55512e520 100644 --- a/codex-rs/core/src/chat_completions.rs +++ b/codex-rs/core/src/chat_completions.rs @@ -25,6 +25,7 @@ use crate::flags::OPENAI_REQUEST_MAX_RETRIES; use crate::flags::OPENAI_STREAM_IDLE_TIMEOUT_MS; use crate::models::ContentItem; use crate::models::ResponseItem; +use crate::openai_tools::create_tools_json_for_chat_completions_api; use crate::util::backoff; /// Implementation for the classic Chat Completions API. This is intentionally @@ -56,17 +57,22 @@ pub(crate) async fn stream_chat_completions( } } + let tools_json = create_tools_json_for_chat_completions_api(prompt, model)?; let payload = json!({ "model": model, "messages": messages, - "stream": true + "stream": true, + "tools": tools_json, }); let base_url = provider.base_url.trim_end_matches('/'); let url = format!("{}/chat/completions", base_url); debug!(url, "POST (chat)"); - trace!("request payload: {}", payload); + trace!( + "request payload: {}", + serde_json::to_string_pretty(&payload).unwrap_or_default() + ); let api_key = provider.api_key()?; let mut attempt = 0; diff --git a/codex-rs/core/src/client.rs b/codex-rs/core/src/client.rs index 72ce845fc8..034cfaec45 100644 --- a/codex-rs/core/src/client.rs +++ b/codex-rs/core/src/client.rs @@ -1,7 +1,5 @@ -use std::collections::BTreeMap; use std::io::BufRead; use std::path::Path; -use std::sync::LazyLock; use std::time::Duration; use bytes::Bytes; @@ -11,7 +9,6 @@ use reqwest::StatusCode; use serde::Deserialize; use serde::Serialize; use serde_json::Value; -use serde_json::json; use tokio::sync::mpsc; use tokio::time::timeout; use tokio_util::io::ReaderStream; @@ -36,71 +33,9 @@ use crate::flags::OPENAI_STREAM_IDLE_TIMEOUT_MS; use crate::model_provider_info::ModelProviderInfo; use crate::model_provider_info::WireApi; use crate::models::ResponseItem; +use crate::openai_tools::create_tools_json_for_responses_api; use crate::util::backoff; -/// When serialized as JSON, this produces a valid "Tool" in the OpenAI -/// Responses API. -#[derive(Debug, Clone, Serialize)] -#[serde(tag = "type")] -enum OpenAiTool { - #[serde(rename = "function")] - Function(ResponsesApiTool), - #[serde(rename = "local_shell")] - LocalShell {}, -} - -#[derive(Debug, Clone, Serialize)] -struct ResponsesApiTool { - name: &'static str, - description: &'static str, - strict: bool, - parameters: JsonSchema, -} - -/// Generic JSON‑Schema subset needed for our tool definitions -#[derive(Debug, Clone, Serialize)] -#[serde(tag = "type", rename_all = "lowercase")] -enum JsonSchema { - String, - Number, - Array { - items: Box, - }, - Object { - properties: BTreeMap, - required: &'static [&'static str], - #[serde(rename = "additionalProperties")] - additional_properties: bool, - }, -} - -/// Tool usage specification -static DEFAULT_TOOLS: LazyLock> = LazyLock::new(|| { - let mut properties = BTreeMap::new(); - properties.insert( - "command".to_string(), - JsonSchema::Array { - items: Box::new(JsonSchema::String), - }, - ); - properties.insert("workdir".to_string(), JsonSchema::String); - properties.insert("timeout".to_string(), JsonSchema::Number); - - vec![OpenAiTool::Function(ResponsesApiTool { - name: "shell", - description: "Runs a shell command, and returns its output.", - strict: false, - parameters: JsonSchema::Object { - properties, - required: &["command"], - additional_properties: false, - }, - })] -}); - -static DEFAULT_CODEX_MODEL_TOOLS: LazyLock> = - LazyLock::new(|| vec![OpenAiTool::LocalShell {}]); - #[derive(Clone)] pub struct ModelClient { model: String, @@ -161,27 +96,8 @@ impl ModelClient { return stream_from_fixture(path).await; } - // Assemble tool list: built-in tools + any extra tools from the prompt. - let default_tools = if self.model.starts_with("codex") { - &DEFAULT_CODEX_MODEL_TOOLS - } else { - &DEFAULT_TOOLS - }; - let mut tools_json = Vec::with_capacity(default_tools.len() + prompt.extra_tools.len()); - for t in default_tools.iter() { - tools_json.push(serde_json::to_value(t)?); - } - tools_json.extend( - prompt - .extra_tools - .clone() - .into_iter() - .map(|(name, tool)| mcp_tool_to_openai_tool(name, tool)), - ); - - debug!("tools_json: {}", serde_json::to_string_pretty(&tools_json)?); - let full_instructions = prompt.get_full_instructions(); + let tools_json = create_tools_json_for_responses_api(prompt, &self.model)?; let payload = Payload { model: &self.model, instructions: &full_instructions, @@ -276,34 +192,6 @@ impl ModelClient { } } -fn mcp_tool_to_openai_tool( - fully_qualified_name: String, - tool: mcp_types::Tool, -) -> serde_json::Value { - let mcp_types::Tool { - description, - mut input_schema, - .. - } = tool; - - // OpenAI models mandate the "properties" field in the schema. The Agents - // SDK fixed this by inserting an empty object for "properties" if it is not - // already present https://github.com/openai/openai-agents-python/issues/449 - // so here we do the same. - if input_schema.properties.is_none() { - input_schema.properties = Some(serde_json::Value::Object(serde_json::Map::new())); - } - - // TODO(mbolin): Change the contract of this function to return - // ResponsesApiTool. - json!({ - "name": fully_qualified_name, - "description": description, - "parameters": input_schema, - "type": "function", - }) -} - #[derive(Debug, Deserialize, Serialize)] struct SseEvent { #[serde(rename = "type")] diff --git a/codex-rs/core/src/lib.rs b/codex-rs/core/src/lib.rs index 8398ff7650..77941a9a51 100644 --- a/codex-rs/core/src/lib.rs +++ b/codex-rs/core/src/lib.rs @@ -27,6 +27,7 @@ mod model_provider_info; pub use model_provider_info::ModelProviderInfo; pub use model_provider_info::WireApi; mod models; +mod openai_tools; mod project_doc; pub mod protocol; mod rollout; diff --git a/codex-rs/core/src/openai_tools.rs b/codex-rs/core/src/openai_tools.rs new file mode 100644 index 0000000000..0cbdcae0d3 --- /dev/null +++ b/codex-rs/core/src/openai_tools.rs @@ -0,0 +1,158 @@ +use serde::Serialize; +use serde_json::json; +use std::collections::BTreeMap; +use std::sync::LazyLock; + +use crate::client_common::Prompt; + +#[derive(Debug, Clone, Serialize)] +pub(crate) struct ResponsesApiTool { + name: &'static str, + description: &'static str, + strict: bool, + parameters: JsonSchema, +} + +/// When serialized as JSON, this produces a valid "Tool" in the OpenAI +/// Responses API. +#[derive(Debug, Clone, Serialize)] +#[serde(tag = "type")] +pub(crate) enum OpenAiTool { + #[serde(rename = "function")] + Function(ResponsesApiTool), + #[serde(rename = "local_shell")] + LocalShell {}, +} + +/// Generic JSON‑Schema subset needed for our tool definitions +#[derive(Debug, Clone, Serialize)] +#[serde(tag = "type", rename_all = "lowercase")] +pub(crate) enum JsonSchema { + String, + Number, + Array { + items: Box, + }, + Object { + properties: BTreeMap, + required: &'static [&'static str], + #[serde(rename = "additionalProperties")] + additional_properties: bool, + }, +} + +/// Tool usage specification +static DEFAULT_TOOLS: LazyLock> = LazyLock::new(|| { + let mut properties = BTreeMap::new(); + properties.insert( + "command".to_string(), + JsonSchema::Array { + items: Box::new(JsonSchema::String), + }, + ); + properties.insert("workdir".to_string(), JsonSchema::String); + properties.insert("timeout".to_string(), JsonSchema::Number); + + vec![OpenAiTool::Function(ResponsesApiTool { + name: "shell", + description: "Runs a shell command, and returns its output.", + strict: false, + parameters: JsonSchema::Object { + properties, + required: &["command"], + additional_properties: false, + }, + })] +}); + +static DEFAULT_CODEX_MODEL_TOOLS: LazyLock> = + LazyLock::new(|| vec![OpenAiTool::LocalShell {}]); + +/// Returns JSON values that are compatible with Function Calling in the +/// Responses API: +/// https://platform.openai.com/docs/guides/function-calling?api-mode=responses +pub(crate) fn create_tools_json_for_responses_api( + prompt: &Prompt, + model: &str, +) -> crate::error::Result> { + // Assemble tool list: built-in tools + any extra tools from the prompt. + let default_tools = if model.starts_with("codex") { + &DEFAULT_CODEX_MODEL_TOOLS + } else { + &DEFAULT_TOOLS + }; + let mut tools_json = Vec::with_capacity(default_tools.len() + prompt.extra_tools.len()); + for t in default_tools.iter() { + tools_json.push(serde_json::to_value(t)?); + } + tools_json.extend( + prompt + .extra_tools + .clone() + .into_iter() + .map(|(name, tool)| mcp_tool_to_openai_tool(name, tool)), + ); + + tracing::debug!("tools_json: {}", serde_json::to_string_pretty(&tools_json)?); + Ok(tools_json) +} + +/// Returns JSON values that are compatible with Function Calling in the +/// Chat Completions API: +/// https://platform.openai.com/docs/guides/function-calling?api-mode=chat +pub(crate) fn create_tools_json_for_chat_completions_api( + prompt: &Prompt, + model: &str, +) -> crate::error::Result> { + // We start with the JSON for the Responses API and than rewrite it to match + // the chat completions tool call format. + let responses_api_tools_json = create_tools_json_for_responses_api(prompt, model)?; + let tools_json = responses_api_tools_json + .into_iter() + .filter_map(|mut tool| { + if tool.get("type") != Some(&serde_json::Value::String("function".to_string())) { + return None; + } + + if let Some(map) = tool.as_object_mut() { + // Remove "type" field as it is not needed in chat completions. + map.remove("type"); + Some(json!({ + "type": "function", + "function": map, + })) + } else { + None + } + }) + .collect::>(); + Ok(tools_json) +} + +fn mcp_tool_to_openai_tool( + fully_qualified_name: String, + tool: mcp_types::Tool, +) -> serde_json::Value { + let mcp_types::Tool { + description, + mut input_schema, + .. + } = tool; + + // OpenAI models mandate the "properties" field in the schema. The Agents + // SDK fixed this by inserting an empty object for "properties" if it is not + // already present https://github.com/openai/openai-agents-python/issues/449 + // so here we do the same. + if input_schema.properties.is_none() { + input_schema.properties = Some(serde_json::Value::Object(serde_json::Map::new())); + } + + // TODO(mbolin): Change the contract of this function to return + // ResponsesApiTool. + json!({ + "name": fully_qualified_name, + "description": description, + "parameters": input_schema, + "type": "function", + }) +} From ae743d56b09f29b3a28a57a6c8aff7565e7b075a Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Fri, 30 May 2025 14:41:55 -0700 Subject: [PATCH 17/41] feat: for `codex exec`, if PROMPT is not specified, read from stdin if not a TTY (#1178) This attempts to make `codex exec` more flexible in how the prompt can be passed: * as before, it can be passed as a single string argument * if `-` is passed as the value, the prompt is read from stdin * if no argument is passed _and stdin is a tty_, prints a warning to stderr that no prompt was specified an exits non-zero. * if no argument is passed _and stdin is NOT a tty_, prints `Reading prompt from stdin...` to stderr to let the user know that Codex will wait until it reads EOF from stdin to proceed. (You can repro this case by doing `yes | just exec` since stdin is not a TTY in that case but it also never reaches EOF). --- codex-rs/exec/src/cli.rs | 6 ++++-- codex-rs/exec/src/lib.rs | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/codex-rs/exec/src/cli.rs b/codex-rs/exec/src/cli.rs index 1c2a9eb8aa..413fd23cb7 100644 --- a/codex-rs/exec/src/cli.rs +++ b/codex-rs/exec/src/cli.rs @@ -45,8 +45,10 @@ pub struct Cli { #[arg(long = "output-last-message")] pub last_message_file: Option, - /// Initial instructions for the agent. - pub prompt: String, + /// Initial instructions for the agent. If not provided as an argument (or + /// if `-` is used), instructions are read from stdin. + #[arg(value_name = "PROMPT")] + pub prompt: Option, } #[derive(Debug, Clone, Copy, Default, PartialEq, Eq, ValueEnum)] diff --git a/codex-rs/exec/src/lib.rs b/codex-rs/exec/src/lib.rs index 8c94fe5dc9..6602213b14 100644 --- a/codex-rs/exec/src/lib.rs +++ b/codex-rs/exec/src/lib.rs @@ -2,6 +2,7 @@ mod cli; mod event_processor; use std::io::IsTerminal; +use std::io::Read; use std::path::Path; use std::path::PathBuf; use std::sync::Arc; @@ -40,6 +41,41 @@ pub async fn run_main(cli: Cli, codex_linux_sandbox_exe: Option) -> any config_overrides, } = cli; + // Determine the prompt based on CLI arg and/or stdin. + let prompt = match prompt { + Some(p) if p != "-" => p, + // Either `-` was passed or no positional arg. + maybe_dash => { + // When no arg (None) **and** stdin is a TTY, bail out early – unless the + // user explicitly forced reading via `-`. + let force_stdin = matches!(maybe_dash.as_deref(), Some("-")); + + if std::io::stdin().is_terminal() && !force_stdin { + eprintln!( + "No prompt provided. Either specify one as an argument or pipe the prompt into stdin." + ); + std::process::exit(1); + } + + // Ensure the user knows we are waiting on stdin, as they may + // have gotten into this state by mistake. If so, and they are not + // writing to stdin, Codex will hang indefinitely, so this should + // help them debug in that case. + if !force_stdin { + eprintln!("Reading prompt from stdin..."); + } + let mut buffer = String::new(); + if let Err(e) = std::io::stdin().read_to_string(&mut buffer) { + eprintln!("Failed to read prompt from stdin: {e}"); + std::process::exit(1); + } else if buffer.trim().is_empty() { + eprintln!("No prompt provided via stdin."); + std::process::exit(1); + } + buffer + } + }; + let (stdout_with_ansi, stderr_with_ansi) = match color { cli::Color::Always => (true, true), cli::Color::Never => (false, false), From cf1d0705389ac4687987acd760f58f126b28b98a Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Fri, 30 May 2025 16:22:10 -0700 Subject: [PATCH 18/41] feat: grab-bag of improvements to `exec` output (#1179) Fixes: * Instantiate `EventProcessor` earlier in `lib.rs` so `print_config_summary()` can be an instance method of it and leverage its various `Style` fields to ensure it honors `with_ansi` properly. * After printing the config summary, print out user's prompt with the heading `User instructions:`. As noted in the comment, now that we can read the instructions via stdin as of #1178, it is helpful to the user to ensure they know what instructions were given to Codex. * Use same colors/bold/italic settings for headers as the TUI, making the output a bit easier to read. --- codex-rs/exec/src/event_processor.rs | 77 +++++++++++++++++----------- codex-rs/exec/src/lib.rs | 8 +-- 2 files changed, 50 insertions(+), 35 deletions(-) diff --git a/codex-rs/exec/src/event_processor.rs b/codex-rs/exec/src/event_processor.rs index 352275bf43..8005980d8e 100644 --- a/codex-rs/exec/src/event_processor.rs +++ b/codex-rs/exec/src/event_processor.rs @@ -1,4 +1,3 @@ -use chrono::Utc; use codex_common::elapsed::format_elapsed; use codex_core::config::Config; use codex_core::protocol::AgentMessageEvent; @@ -37,11 +36,13 @@ pub(crate) struct EventProcessor { // using .style() with one of these fields. If you need a new style, add a // new field here. bold: Style, + italic: Style, dimmed: Style, magenta: Style, red: Style, green: Style, + cyan: Style, } impl EventProcessor { @@ -55,10 +56,12 @@ impl EventProcessor { call_id_to_command, call_id_to_patch, bold: Style::new().bold(), + italic: Style::new().italic(), dimmed: Style::new().dimmed(), magenta: Style::new().magenta(), red: Style::new().red(), green: Style::new().green(), + cyan: Style::new().cyan(), call_id_to_tool_call, } } else { @@ -66,10 +69,12 @@ impl EventProcessor { call_id_to_command, call_id_to_patch, bold: Style::new(), + italic: Style::new(), dimmed: Style::new(), magenta: Style::new(), red: Style::new(), green: Style::new(), + cyan: Style::new(), call_id_to_tool_call, } } @@ -94,43 +99,47 @@ struct PatchApplyBegin { auto_approved: bool, } +#[macro_export] macro_rules! ts_println { ($($arg:tt)*) => {{ - let now = Utc::now(); + let now = chrono::Utc::now(); let formatted = now.format("%Y-%m-%dT%H:%M:%S").to_string(); print!("[{}] ", formatted); println!($($arg)*); }}; } -/// Print a concise summary of the effective configuration that will be used -/// for the session. This mirrors the information shown in the TUI welcome -/// screen. -pub(crate) fn print_config_summary(config: &Config, with_ansi: bool) { - let bold = if with_ansi { - Style::new().bold() - } else { - Style::new() - }; - - ts_println!("OpenAI Codex (research preview)\n--------"); - - let entries = vec![ - ("workdir", config.cwd.display().to_string()), - ("model", config.model.clone()), - ("provider", config.model_provider_id.clone()), - ("approval", format!("{:?}", config.approval_policy)), - ("sandbox", format!("{:?}", config.sandbox_policy)), - ]; - - for (key, value) in entries { - println!("{} {}", format!("{key}: ").style(bold), value); - } +impl EventProcessor { + /// Print a concise summary of the effective configuration that will be used + /// for the session. This mirrors the information shown in the TUI welcome + /// screen. + pub(crate) fn print_config_summary(&mut self, config: &Config, prompt: &str) { + ts_println!("OpenAI Codex (research preview)\n--------"); + + let entries = vec![ + ("workdir", config.cwd.display().to_string()), + ("model", config.model.clone()), + ("provider", config.model_provider_id.clone()), + ("approval", format!("{:?}", config.approval_policy)), + ("sandbox", format!("{:?}", config.sandbox_policy)), + ]; + + for (key, value) in entries { + println!("{} {}", format!("{key}: ").style(self.bold), value); + } - println!("--------\n"); -} + println!("--------"); + + // Echo the prompt that will be sent to the agent so it is visible in the + // transcript/logs before any events come in. Note the prompt may have been + // read from stdin, so it may not be visible in the terminal otherwise. + ts_println!( + "{}\n{}", + "User instructions:".style(self.bold).style(self.cyan), + prompt + ); + } -impl EventProcessor { pub(crate) fn process_event(&mut self, event: Event) { let Event { id: _, msg } = event; match msg { @@ -145,8 +154,10 @@ impl EventProcessor { // Ignore. } EventMsg::AgentMessage(AgentMessageEvent { message }) => { - let prefix = "Agent message:".style(self.bold); - ts_println!("{prefix} {message}"); + ts_println!( + "{}\n{message}", + "codex".style(self.bold).style(self.magenta) + ); } EventMsg::ExecCommandBegin(ExecCommandBeginEvent { call_id, @@ -394,7 +405,11 @@ impl EventProcessor { // Should we exit? } EventMsg::AgentReasoning(agent_reasoning_event) => { - println!("thinking: {}", agent_reasoning_event.text); + ts_println!( + "{}\n{}", + "thinking".style(self.italic).style(self.magenta), + agent_reasoning_event.text + ); } EventMsg::SessionConfigured(session_configured_event) => { let SessionConfiguredEvent { diff --git a/codex-rs/exec/src/lib.rs b/codex-rs/exec/src/lib.rs index 6602213b14..e203c2f161 100644 --- a/codex-rs/exec/src/lib.rs +++ b/codex-rs/exec/src/lib.rs @@ -20,7 +20,6 @@ use codex_core::protocol::SandboxPolicy; use codex_core::protocol::TaskCompleteEvent; use codex_core::util::is_inside_git_repo; use event_processor::EventProcessor; -use event_processor::print_config_summary; use tracing::debug; use tracing::error; use tracing::info; @@ -113,8 +112,10 @@ pub async fn run_main(cli: Cli, codex_linux_sandbox_exe: Option) -> any }; let config = Config::load_with_cli_overrides(cli_kv_overrides, overrides)?; - // Print the effective configuration so users can see what Codex is using. - print_config_summary(&config, stdout_with_ansi); + let mut event_processor = EventProcessor::create_with_ansi(stdout_with_ansi); + // Print the effective configuration and prompt so users can see what Codex + // is using. + event_processor.print_config_summary(&config, &prompt); if !skip_git_repo_check && !is_inside_git_repo(&config) { eprintln!("Not inside a Git repo and --skip-git-repo-check was not specified."); @@ -204,7 +205,6 @@ pub async fn run_main(cli: Cli, codex_linux_sandbox_exe: Option) -> any info!("Sent prompt with event ID: {initial_prompt_task_id}"); // Run the loop until the task is complete. - let mut event_processor = EventProcessor::create_with_ansi(stdout_with_ansi); while let Some(event) = rx.recv().await { let (is_last_event, last_assistant_message) = match &event.msg { EventMsg::TaskComplete(TaskCompleteEvent { last_agent_message }) => { From 4f3d294762e7351268d4670812a24cd4a876a73a Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Fri, 30 May 2025 16:27:37 -0700 Subject: [PATCH 19/41] feat: dim the timestamp in the exec output (#1180) This required changing `ts_println!()` to take `$self:ident`, which is a bit more verbose, but the usability improvement seems worth it. Also eliminated an unnecessary `.to_string()` while here. --- codex-rs/exec/src/event_processor.rs | 30 ++++++++++++++++++---------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/codex-rs/exec/src/event_processor.rs b/codex-rs/exec/src/event_processor.rs index 8005980d8e..57a6cbc9ac 100644 --- a/codex-rs/exec/src/event_processor.rs +++ b/codex-rs/exec/src/event_processor.rs @@ -99,12 +99,13 @@ struct PatchApplyBegin { auto_approved: bool, } +// Timestamped println helper. The timestamp is styled with self.dimmed. #[macro_export] macro_rules! ts_println { - ($($arg:tt)*) => {{ + ($self:ident, $($arg:tt)*) => {{ let now = chrono::Utc::now(); - let formatted = now.format("%Y-%m-%dT%H:%M:%S").to_string(); - print!("[{}] ", formatted); + let formatted = now.format("[%Y-%m-%dT%H:%M:%S]"); + print!("{} ", formatted.style($self.dimmed)); println!($($arg)*); }}; } @@ -114,7 +115,7 @@ impl EventProcessor { /// for the session. This mirrors the information shown in the TUI welcome /// screen. pub(crate) fn print_config_summary(&mut self, config: &Config, prompt: &str) { - ts_println!("OpenAI Codex (research preview)\n--------"); + ts_println!(self, "OpenAI Codex (research preview)\n--------"); let entries = vec![ ("workdir", config.cwd.display().to_string()), @@ -134,6 +135,7 @@ impl EventProcessor { // transcript/logs before any events come in. Note the prompt may have been // read from stdin, so it may not be visible in the terminal otherwise. ts_println!( + self, "{}\n{}", "User instructions:".style(self.bold).style(self.cyan), prompt @@ -145,16 +147,17 @@ impl EventProcessor { match msg { EventMsg::Error(ErrorEvent { message }) => { let prefix = "ERROR:".style(self.red); - ts_println!("{prefix} {message}"); + ts_println!(self, "{prefix} {message}"); } EventMsg::BackgroundEvent(BackgroundEventEvent { message }) => { - ts_println!("{}", message.style(self.dimmed)); + ts_println!(self, "{}", message.style(self.dimmed)); } EventMsg::TaskStarted | EventMsg::TaskComplete(_) => { // Ignore. } EventMsg::AgentMessage(AgentMessageEvent { message }) => { ts_println!( + self, "{}\n{message}", "codex".style(self.bold).style(self.magenta) ); @@ -172,6 +175,7 @@ impl EventProcessor { }, ); ts_println!( + self, "{} {} in {}", "exec".style(self.magenta), escape_command(&command).style(self.bold), @@ -207,11 +211,11 @@ impl EventProcessor { match exit_code { 0 => { let title = format!("{call} succeeded{duration}:"); - ts_println!("{}", title.style(self.green)); + ts_println!(self, "{}", title.style(self.green)); } _ => { let title = format!("{call} exited {exit_code}{duration}:"); - ts_println!("{}", title.style(self.red)); + ts_println!(self, "{}", title.style(self.red)); } } println!("{}", truncated_output.style(self.dimmed)); @@ -248,6 +252,7 @@ impl EventProcessor { ); ts_println!( + self, "{} {}", "tool".style(self.magenta), invocation.style(self.bold), @@ -274,7 +279,7 @@ impl EventProcessor { let title_style = if is_success { self.green } else { self.red }; let title = format!("{invocation} {status_str}{duration}:"); - ts_println!("{}", title.style(title_style)); + ts_println!(self, "{}", title.style(title_style)); if let Ok(res) = result { let val: serde_json::Value = res.into(); @@ -302,6 +307,7 @@ impl EventProcessor { ); ts_println!( + self, "{} auto_approved={}:", "apply_patch".style(self.magenta), auto_approved, @@ -393,7 +399,7 @@ impl EventProcessor { }; let title = format!("{label} exited {exit_code}{duration}:"); - ts_println!("{}", title.style(title_style)); + ts_println!(self, "{}", title.style(title_style)); for line in output.lines() { println!("{}", line.style(self.dimmed)); } @@ -406,6 +412,7 @@ impl EventProcessor { } EventMsg::AgentReasoning(agent_reasoning_event) => { ts_println!( + self, "{}\n{}", "thinking".style(self.italic).style(self.magenta), agent_reasoning_event.text @@ -420,12 +427,13 @@ impl EventProcessor { } = session_configured_event; ts_println!( + self, "{} {}", "codex session".style(self.magenta).style(self.bold), session_id.to_string().style(self.dimmed) ); - ts_println!("model: {}", model); + ts_println!(self, "model: {}", model); println!(); } EventMsg::GetHistoryEntryResponse(_) => { From e81327e5f4ad945440960093c9b55e3da05cc32d Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Fri, 30 May 2025 23:14:56 -0700 Subject: [PATCH 20/41] feat: add hide_agent_reasoning config option (#1181) This PR introduces a `hide_agent_reasoning` config option (that defaults to `false`) that users can enable to make the output less verbose by suppressing reasoning output. To test, verified that this includes agent reasoning in the output: ``` echo hello | just exec ``` whereas this does not: ``` echo hello | just exec --config hide_agent_reasoning=false ``` --- codex-rs/config.md | 10 ++++++++++ codex-rs/core/src/config.rs | 14 ++++++++++++++ codex-rs/exec/src/event_processor.rs | 21 ++++++++++++++------- codex-rs/exec/src/lib.rs | 3 ++- codex-rs/tui/src/chatwidget.rs | 8 +++++--- 5 files changed, 45 insertions(+), 11 deletions(-) diff --git a/codex-rs/config.md b/codex-rs/config.md index a1caacfcbc..416eeb4144 100644 --- a/codex-rs/config.md +++ b/codex-rs/config.md @@ -354,6 +354,16 @@ Note this is **not** a general editor setting (like `$EDITOR`), as it only accep Currently, `"vscode"` is the default, though Codex does not verify VS Code is installed. As such, `file_opener` may default to `"none"` or something else in the future. +## hide_agent_reasoning + +Codex intermittently emits "reasoning" events that show the model’s internal "thinking" before it produces a final answer. Some users may find these events distracting, especially in CI logs or minimal terminal output. + +Setting `hide_agent_reasoning` to `true` suppresses these events in **both** the TUI as well as the headless `exec` sub-command: + +```toml +hide_agent_reasoning = true # defaults to false +``` + ## project_doc_max_bytes Maximum number of bytes to read from an `AGENTS.md` file to include in the instructions sent with the first turn of a session. Defaults to 32 KiB. diff --git a/codex-rs/core/src/config.rs b/codex-rs/core/src/config.rs index b6871da153..d948ddb916 100644 --- a/codex-rs/core/src/config.rs +++ b/codex-rs/core/src/config.rs @@ -42,6 +42,11 @@ pub struct Config { pub shell_environment_policy: ShellEnvironmentPolicy, + /// When `true`, `AgentReasoning` events emitted by the backend will be + /// suppressed from the frontend output. This can reduce visual noise when + /// users are only interested in the final agent responses. + pub hide_agent_reasoning: bool, + /// Disable server-side response storage (sends the full conversation /// context with every request). Currently necessary for OpenAI customers /// who have opted into Zero Data Retention (ZDR). @@ -272,6 +277,10 @@ pub struct ConfigToml { /// Collection of settings that are specific to the TUI. pub tui: Option, + + /// When set to `true`, `AgentReasoning` events will be hidden from the + /// UI/output. Defaults to `false`. + pub hide_agent_reasoning: Option, } fn deserialize_sandbox_permissions<'de, D>( @@ -433,6 +442,8 @@ impl Config { file_opener: cfg.file_opener.unwrap_or(UriBasedFileOpener::VsCode), tui: cfg.tui.unwrap_or_default(), codex_linux_sandbox_exe, + + hide_agent_reasoning: cfg.hide_agent_reasoning.unwrap_or(false), }; Ok(config) } @@ -774,6 +785,7 @@ disable_response_storage = true file_opener: UriBasedFileOpener::VsCode, tui: Tui::default(), codex_linux_sandbox_exe: None, + hide_agent_reasoning: false, }, o3_profile_config ); @@ -813,6 +825,7 @@ disable_response_storage = true file_opener: UriBasedFileOpener::VsCode, tui: Tui::default(), codex_linux_sandbox_exe: None, + hide_agent_reasoning: false, }; assert_eq!(expected_gpt3_profile_config, gpt3_profile_config); @@ -867,6 +880,7 @@ disable_response_storage = true file_opener: UriBasedFileOpener::VsCode, tui: Tui::default(), codex_linux_sandbox_exe: None, + hide_agent_reasoning: false, }; assert_eq!(expected_zdr_profile_config, zdr_profile_config); diff --git a/codex-rs/exec/src/event_processor.rs b/codex-rs/exec/src/event_processor.rs index 57a6cbc9ac..89ad7d7cde 100644 --- a/codex-rs/exec/src/event_processor.rs +++ b/codex-rs/exec/src/event_processor.rs @@ -43,10 +43,13 @@ pub(crate) struct EventProcessor { red: Style, green: Style, cyan: Style, + + /// Whether to include `AgentReasoning` events in the output. + show_agent_reasoning: bool, } impl EventProcessor { - pub(crate) fn create_with_ansi(with_ansi: bool) -> Self { + pub(crate) fn create_with_ansi(with_ansi: bool, show_agent_reasoning: bool) -> Self { let call_id_to_command = HashMap::new(); let call_id_to_patch = HashMap::new(); let call_id_to_tool_call = HashMap::new(); @@ -63,6 +66,7 @@ impl EventProcessor { green: Style::new().green(), cyan: Style::new().cyan(), call_id_to_tool_call, + show_agent_reasoning, } } else { Self { @@ -76,6 +80,7 @@ impl EventProcessor { green: Style::new(), cyan: Style::new(), call_id_to_tool_call, + show_agent_reasoning, } } } @@ -411,12 +416,14 @@ impl EventProcessor { // Should we exit? } EventMsg::AgentReasoning(agent_reasoning_event) => { - ts_println!( - self, - "{}\n{}", - "thinking".style(self.italic).style(self.magenta), - agent_reasoning_event.text - ); + if self.show_agent_reasoning { + ts_println!( + self, + "{}\n{}", + "thinking".style(self.italic).style(self.magenta), + agent_reasoning_event.text + ); + } } EventMsg::SessionConfigured(session_configured_event) => { let SessionConfiguredEvent { diff --git a/codex-rs/exec/src/lib.rs b/codex-rs/exec/src/lib.rs index e203c2f161..925e25d670 100644 --- a/codex-rs/exec/src/lib.rs +++ b/codex-rs/exec/src/lib.rs @@ -112,7 +112,8 @@ pub async fn run_main(cli: Cli, codex_linux_sandbox_exe: Option) -> any }; let config = Config::load_with_cli_overrides(cli_kv_overrides, overrides)?; - let mut event_processor = EventProcessor::create_with_ansi(stdout_with_ansi); + let mut event_processor = + EventProcessor::create_with_ansi(stdout_with_ansi, !config.hide_agent_reasoning); // Print the effective configuration and prompt so users can see what Codex // is using. event_processor.print_config_summary(&config, &prompt); diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index 4819be3809..63f3bc727a 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -239,9 +239,11 @@ impl ChatWidget<'_> { self.request_redraw(); } EventMsg::AgentReasoning(AgentReasoningEvent { text }) => { - self.conversation_history - .add_agent_reasoning(&self.config, text); - self.request_redraw(); + if !self.config.hide_agent_reasoning { + self.conversation_history + .add_agent_reasoning(&self.config, text); + self.request_redraw(); + } } EventMsg::TaskStarted => { self.bottom_pane.set_task_running(true); From 1159eaf04f26a95d2ffdfcffb6b7368e47996f4f Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Fri, 30 May 2025 23:24:36 -0700 Subject: [PATCH 21/41] feat: show the version when starting Codex (#1182) The TypeScript version of the CLI shows the version when it starts up, which is helpful when users share screenshots (and nice to know, as a user). --- codex-rs/exec/src/event_processor.rs | 7 ++++++- codex-rs/tui/src/history_cell.rs | 3 +++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/codex-rs/exec/src/event_processor.rs b/codex-rs/exec/src/event_processor.rs index 89ad7d7cde..5462736b5f 100644 --- a/codex-rs/exec/src/event_processor.rs +++ b/codex-rs/exec/src/event_processor.rs @@ -120,7 +120,12 @@ impl EventProcessor { /// for the session. This mirrors the information shown in the TUI welcome /// screen. pub(crate) fn print_config_summary(&mut self, config: &Config, prompt: &str) { - ts_println!(self, "OpenAI Codex (research preview)\n--------"); + const VERSION: &str = env!("CARGO_PKG_VERSION"); + ts_println!( + self, + "OpenAI Codex v{} (research preview)\n--------", + VERSION + ); let entries = vec![ ("workdir", config.cwd.display().to_string()), diff --git a/codex-rs/tui/src/history_cell.rs b/codex-rs/tui/src/history_cell.rs index 41c2049313..b41c8ac62b 100644 --- a/codex-rs/tui/src/history_cell.rs +++ b/codex-rs/tui/src/history_cell.rs @@ -130,10 +130,13 @@ impl HistoryCell { history_entry_count: _, } = event; if is_first_event { + const VERSION: &str = env!("CARGO_PKG_VERSION"); + let mut lines: Vec> = vec![ Line::from(vec![ "OpenAI ".into(), "Codex".bold(), + format!(" v{}", VERSION).into(), " (research preview)".dim(), ]), Line::from(""), From fccf5f322128f61e3229718d827046e7ec0fc868 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Fri, 30 May 2025 23:49:48 -0700 Subject: [PATCH 22/41] fix: disable agent reasoning output by default in the GitHub Action (#1183) --- .github/actions/codex/action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/codex/action.yml b/.github/actions/codex/action.yml index 715423d06a..eb44930200 100644 --- a/.github/actions/codex/action.yml +++ b/.github/actions/codex/action.yml @@ -15,14 +15,14 @@ inputs: codex_args: description: "A whitespace-delimited list of arguments to pass to Codex. Due to limitations in YAML, arguments with spaces are not supported. For more complex configurations, use the `codex_home` input." required: false - default: "--full-auto" + default: "--config hide_agent_reasoning=false --full-auto" codex_home: description: "Value to use as the CODEX_HOME environment variable when running Codex." required: false codex_release_tag: description: "The release tag of the Codex model to run." required: false - default: "codex-rs-d519bd8bbd1e1fd9efdc5d68cf7bebdec0dd0f28-1-rust-v0.0.2505270918" + default: "codex-rs-ca8e97fcbcb991e542b8689f2d4eab9d30c399d6-1-rust-v0.0.2505302325" runs: using: "composite" From 1410ae95caf1f58bf72942f79fdc33f277e2451b Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Fri, 30 May 2025 23:57:05 -0700 Subject: [PATCH 23/41] fix: set `--config hide_agent_reasoning=true` in the GitHub Action (#1185) Whoops, I had this flipped in https://github.com/openai/codex/pull/1183. --- .github/actions/codex/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/codex/action.yml b/.github/actions/codex/action.yml index eb44930200..f0af1cb3e7 100644 --- a/.github/actions/codex/action.yml +++ b/.github/actions/codex/action.yml @@ -15,7 +15,7 @@ inputs: codex_args: description: "A whitespace-delimited list of arguments to pass to Codex. Due to limitations in YAML, arguments with spaces are not supported. For more complex configurations, use the `codex_home` input." required: false - default: "--config hide_agent_reasoning=false --full-auto" + default: "--config hide_agent_reasoning=true --full-auto" codex_home: description: "Value to use as the CODEX_HOME environment variable when running Codex." required: false From 7896b1089dbf702dd07929910504e9558a20d085 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Sat, 31 May 2025 10:30:50 -0700 Subject: [PATCH 24/41] chore: update the WORKFLOW_URL in install_native_deps.sh to the latest release (#1190) --- codex-cli/scripts/install_native_deps.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codex-cli/scripts/install_native_deps.sh b/codex-cli/scripts/install_native_deps.sh index c1697fb5fe..ff434d2e58 100755 --- a/codex-cli/scripts/install_native_deps.sh +++ b/codex-cli/scripts/install_native_deps.sh @@ -65,7 +65,7 @@ mkdir -p "$BIN_DIR" # Until we start publishing stable GitHub releases, we have to grab the binaries # from the GitHub Action that created them. Update the URL below to point to the # appropriate workflow run: -WORKFLOW_URL="https://github.com/openai/codex/actions/runs/15334411824" +WORKFLOW_URL="https://github.com/openai/codex/actions/runs/15361005231" WORKFLOW_ID="${WORKFLOW_URL##*/}" ARTIFACTS_DIR="$(mktemp -d)" From e40f86b446870c4f8873c6e0fef7680649437be6 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Mon, 2 Jun 2025 13:31:33 -0700 Subject: [PATCH 25/41] chore: logging cleanup (#1196) Update what we log to make `RUST_LOG=debug` a bit easier to work with. --- [//]: # (BEGIN SAPLING FOOTER) Stack created with [Sapling](https://sapling-scm.com). Best reviewed with [ReviewStack](https://reviewstack.dev/openai/codex/pull/1196). * #1167 * __->__ #1196 --- codex-rs/core/src/client.rs | 16 ++++++++++++++-- codex-rs/core/src/openai_tools.rs | 1 - 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/codex-rs/core/src/client.rs b/codex-rs/core/src/client.rs index 034cfaec45..6eb20149a5 100644 --- a/codex-rs/core/src/client.rs +++ b/codex-rs/core/src/client.rs @@ -117,8 +117,7 @@ impl ModelClient { let base_url = self.provider.base_url.clone(); let base_url = base_url.trim_end_matches('/'); let url = format!("{}/responses", base_url); - debug!(url, "POST"); - trace!("request payload: {}", serde_json::to_string(&payload)?); + trace!("POST to {url}: {}", serde_json::to_string(&payload)?); let mut attempt = 0; loop { @@ -303,6 +302,19 @@ where }; }; } + "response.content_part.done" + | "response.created" + | "response.function_call_arguments.delta" + | "response.in_progress" + | "response.output_item.added" + | "response.output_text.delta" + | "response.output_text.done" + | "response.reasoning_summary_part.added" + | "response.reasoning_summary_text.delta" + | "response.reasoning_summary_text.done" => { + // Currently, we ignore these events, but we handle them + // separately to skip the logging message in the `other` case. + } other => debug!(other, "sse event"), } } diff --git a/codex-rs/core/src/openai_tools.rs b/codex-rs/core/src/openai_tools.rs index 0cbdcae0d3..ef12a629b6 100644 --- a/codex-rs/core/src/openai_tools.rs +++ b/codex-rs/core/src/openai_tools.rs @@ -93,7 +93,6 @@ pub(crate) fn create_tools_json_for_responses_api( .map(|(name, tool)| mcp_tool_to_openai_tool(name, tool)), ); - tracing::debug!("tools_json: {}", serde_json::to_string_pretty(&tools_json)?); Ok(tools_json) } From d7245cbbc9d8ff5446da45e5951761103492476d Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Mon, 2 Jun 2025 13:47:51 -0700 Subject: [PATCH 26/41] fix: chat completions API now also passes tools along (#1167) Prior to this PR, there were two big misses in `chat_completions.rs`: 1. The loop in `stream_chat_completions()` was only including items of type `ResponseItem::Message` when building up the `"messages"` JSON for the `POST` request to the `chat/completions` endpoint. This fixes things by ensuring other variants (`FunctionCall`, `LocalShellCall`, and `FunctionCallOutput`) are included, as well. 2. In `process_chat_sse()`, we were not recording tool calls and were only emitting items of type `ResponseEvent::OutputItemDone(ResponseItem::Message)` to the stream. Now we introduce `FunctionCallState`, which is used to accumulate the `delta`s of type `tool_calls`, so we can ultimately emit a `ResponseItem::FunctionCall`, when appropriate. While function calling now appears to work for chat completions with my local testing, I believe that there are still edge cases that are not covered and that this codepath would benefit from a battery of integration tests. (As part of that further cleanup, we should also work to support streaming responses in the UI.) The other important part of this PR is some cleanup in `core/src/codex.rs`. In particular, it was hard to reason about how `run_task()` was building up the list of messages to include in a request across the various cases: - Responses API - Chat Completions API - Responses API used in concert with ZDR I like to think things are a bit cleaner now where: - `zdr_transcript` (if present) contains all messages in the history of the conversation, which includes function call outputs that have not been sent back to the model yet - `pending_input` includes any messages the user has submitted while the turn is in flight that need to be injected as part of the next `POST` to the model - `input_for_next_turn` includes the tool call outputs that have not been sent back to the model yet --- codex-rs/core/src/chat_completions.rs | 205 +++++++++++++++++++++----- codex-rs/core/src/codex.rs | 169 ++++++++++++++++----- 2 files changed, 301 insertions(+), 73 deletions(-) diff --git a/codex-rs/core/src/chat_completions.rs b/codex-rs/core/src/chat_completions.rs index f55512e520..416baafc42 100644 --- a/codex-rs/core/src/chat_completions.rs +++ b/codex-rs/core/src/chat_completions.rs @@ -28,8 +28,7 @@ use crate::models::ResponseItem; use crate::openai_tools::create_tools_json_for_chat_completions_api; use crate::util::backoff; -/// Implementation for the classic Chat Completions API. This is intentionally -/// minimal: we only stream back plain assistant text. +/// Implementation for the classic Chat Completions API. pub(crate) async fn stream_chat_completions( prompt: &Prompt, model: &str, @@ -43,17 +42,67 @@ pub(crate) async fn stream_chat_completions( messages.push(json!({"role": "system", "content": full_instructions})); for item in &prompt.input { - if let ResponseItem::Message { role, content } = item { - let mut text = String::new(); - for c in content { - match c { - ContentItem::InputText { text: t } | ContentItem::OutputText { text: t } => { - text.push_str(t); + match item { + ResponseItem::Message { role, content } => { + let mut text = String::new(); + for c in content { + match c { + ContentItem::InputText { text: t } + | ContentItem::OutputText { text: t } => { + text.push_str(t); + } + _ => {} } - _ => {} } + messages.push(json!({"role": role, "content": text})); + } + ResponseItem::FunctionCall { + name, + arguments, + call_id, + } => { + messages.push(json!({ + "role": "assistant", + "content": null, + "tool_calls": [{ + "id": call_id, + "type": "function", + "function": { + "name": name, + "arguments": arguments, + } + }] + })); + } + ResponseItem::LocalShellCall { + id, + call_id: _, + status, + action, + } => { + // Confirm with API team. + messages.push(json!({ + "role": "assistant", + "content": null, + "tool_calls": [{ + "id": id.clone().unwrap_or_else(|| "".to_string()), + "type": "local_shell_call", + "status": status, + "action": action, + }] + })); + } + ResponseItem::FunctionCallOutput { call_id, output } => { + messages.push(json!({ + "role": "tool", + "tool_call_id": call_id, + "content": output.content, + })); + } + ResponseItem::Reasoning { .. } | ResponseItem::Other => { + // Omit these items from the conversation history. + continue; } - messages.push(json!({"role": role, "content": text})); } } @@ -68,9 +117,8 @@ pub(crate) async fn stream_chat_completions( let base_url = provider.base_url.trim_end_matches('/'); let url = format!("{}/chat/completions", base_url); - debug!(url, "POST (chat)"); - trace!( - "request payload: {}", + debug!( + "POST to {url}: {}", serde_json::to_string_pretty(&payload).unwrap_or_default() ); @@ -140,6 +188,21 @@ where let idle_timeout = *OPENAI_STREAM_IDLE_TIMEOUT_MS; + // State to accumulate a function call across streaming chunks. + // OpenAI may split the `arguments` string over multiple `delta` events + // until the chunk whose `finish_reason` is `tool_calls` is emitted. We + // keep collecting the pieces here and forward a single + // `ResponseItem::FunctionCall` once the call is complete. + #[derive(Default)] + struct FunctionCallState { + name: Option, + arguments: String, + call_id: Option, + active: bool, + } + + let mut fn_call_state = FunctionCallState::default(); + loop { let sse = match timeout(idle_timeout, stream.next()).await { Ok(Some(Ok(ev))) => ev, @@ -179,23 +242,89 @@ where Ok(v) => v, Err(_) => continue, }; + trace!("chat_completions received SSE chunk: {chunk:?}"); + + let choice_opt = chunk.get("choices").and_then(|c| c.get(0)); + + if let Some(choice) = choice_opt { + // Handle assistant content tokens. + if let Some(content) = choice + .get("delta") + .and_then(|d| d.get("content")) + .and_then(|c| c.as_str()) + { + let item = ResponseItem::Message { + role: "assistant".to_string(), + content: vec![ContentItem::OutputText { + text: content.to_string(), + }], + }; + + let _ = tx_event.send(Ok(ResponseEvent::OutputItemDone(item))).await; + } + + // Handle streaming function / tool calls. + if let Some(tool_calls) = choice + .get("delta") + .and_then(|d| d.get("tool_calls")) + .and_then(|tc| tc.as_array()) + { + if let Some(tool_call) = tool_calls.first() { + // Mark that we have an active function call in progress. + fn_call_state.active = true; + + // Extract call_id if present. + if let Some(id) = tool_call.get("id").and_then(|v| v.as_str()) { + fn_call_state.call_id.get_or_insert_with(|| id.to_string()); + } + + // Extract function details if present. + if let Some(function) = tool_call.get("function") { + if let Some(name) = function.get("name").and_then(|n| n.as_str()) { + fn_call_state.name.get_or_insert_with(|| name.to_string()); + } + + if let Some(args_fragment) = + function.get("arguments").and_then(|a| a.as_str()) + { + fn_call_state.arguments.push_str(args_fragment); + } + } + } + } + + // Emit end-of-turn when finish_reason signals completion. + if let Some(finish_reason) = choice.get("finish_reason").and_then(|v| v.as_str()) { + match finish_reason { + "tool_calls" if fn_call_state.active => { + // Build the FunctionCall response item. + let item = ResponseItem::FunctionCall { + name: fn_call_state.name.clone().unwrap_or_else(|| "".to_string()), + arguments: fn_call_state.arguments.clone(), + call_id: fn_call_state.call_id.clone().unwrap_or_else(String::new), + }; + + // Emit it downstream. + let _ = tx_event.send(Ok(ResponseEvent::OutputItemDone(item))).await; + } + "stop" => { + // Regular turn without tool-call. + } + _ => {} + } - let content_opt = chunk - .get("choices") - .and_then(|c| c.get(0)) - .and_then(|c| c.get("delta")) - .and_then(|d| d.get("content")) - .and_then(|c| c.as_str()); - - if let Some(content) = content_opt { - let item = ResponseItem::Message { - role: "assistant".to_string(), - content: vec![ContentItem::OutputText { - text: content.to_string(), - }], - }; - - let _ = tx_event.send(Ok(ResponseEvent::OutputItemDone(item))).await; + // Emit Completed regardless of reason so the agent can advance. + let _ = tx_event + .send(Ok(ResponseEvent::Completed { + response_id: String::new(), + })) + .await; + + // Prepare for potential next turn (should not happen in same stream). + // fn_call_state = FunctionCallState::default(); + + return; // End processing for this SSE stream. + } } } } @@ -242,9 +371,14 @@ where Poll::Ready(None) => return Poll::Ready(None), Poll::Ready(Some(Err(e))) => return Poll::Ready(Some(Err(e))), Poll::Ready(Some(Ok(ResponseEvent::OutputItemDone(item)))) => { - // Accumulate *assistant* text but do not emit yet. - if let crate::models::ResponseItem::Message { role, content } = &item { - if role == "assistant" { + // If this is an incremental assistant message chunk, accumulate but + // do NOT emit yet. Forward any other item (e.g. FunctionCall) right + // away so downstream consumers see it. + + let is_assistant_delta = matches!(&item, crate::models::ResponseItem::Message { role, .. } if role == "assistant"); + + if is_assistant_delta { + if let crate::models::ResponseItem::Message { content, .. } = &item { if let Some(text) = content.iter().find_map(|c| match c { crate::models::ContentItem::OutputText { text } => Some(text), _ => None, @@ -252,10 +386,13 @@ where this.cumulative.push_str(text); } } + + // Swallow partial assistant chunk; keep polling. + continue; } - // Swallow partial event; keep polling. - continue; + // Not an assistant message – forward immediately. + return Poll::Ready(Some(Ok(ResponseEvent::OutputItemDone(item)))); } Poll::Ready(Some(Ok(ResponseEvent::Completed { response_id }))) => { if !this.cumulative.is_empty() { diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index 2699a9ce78..01ff459f65 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -20,6 +20,7 @@ use codex_apply_patch::MaybeApplyPatchVerified; use codex_apply_patch::maybe_parse_apply_patch_verified; use codex_apply_patch::print_summary; use futures::prelude::*; +use mcp_types::CallToolResult; use serde::Serialize; use serde_json; use tokio::sync::Notify; @@ -295,6 +296,17 @@ impl Session { state.approved_commands.insert(cmd); } + /// Records items to both the rollout and the chat completions/ZDR + /// transcript, if enabled. + async fn record_conversation_items(&self, items: &[ResponseItem]) { + debug!("Recording items for conversation: {items:?}"); + self.record_rollout_items(items).await; + + if let Some(transcript) = self.state.lock().unwrap().zdr_transcript.as_mut() { + transcript.record_items(items); + } + } + /// Append the given items to the session's rollout transcript (if enabled) /// and persist them to disk. async fn record_rollout_items(&self, items: &[ResponseItem]) { @@ -388,7 +400,7 @@ impl Session { tool: &str, arguments: Option, timeout: Option, - ) -> anyhow::Result { + ) -> anyhow::Result { self.mcp_connection_manager .call_tool(server, tool, arguments, timeout) .await @@ -760,6 +772,19 @@ async fn submission_loop( debug!("Agent loop exited"); } +/// Takes a user message as input and runs a loop where, at each turn, the model +/// replies with either: +/// +/// - requested function calls +/// - an assistant message +/// +/// While it is possible for the model to return multiple of these items in a +/// single turn, in practice, we generally one item per turn: +/// +/// - If the model requests a function call, we execute it and send the output +/// back to the model in the next turn. +/// - If the model sends only an assistant message, we record it in the +/// conversation history and consider the task complete. async fn run_task(sess: Arc, sub_id: String, input: Vec) { if input.is_empty() { return; @@ -772,10 +797,14 @@ async fn run_task(sess: Arc, sub_id: String, input: Vec) { return; } - let mut pending_response_input: Vec = vec![ResponseInputItem::from(input)]; + let initial_input_for_turn = ResponseInputItem::from(input); + sess.record_conversation_items(&[initial_input_for_turn.clone().into()]) + .await; + + let mut input_for_next_turn: Vec = vec![initial_input_for_turn]; let last_agent_message: Option; loop { - let mut net_new_turn_input = pending_response_input + let mut net_new_turn_input = input_for_next_turn .drain(..) .map(ResponseItem::from) .collect::>(); @@ -783,11 +812,12 @@ async fn run_task(sess: Arc, sub_id: String, input: Vec) { // Note that pending_input would be something like a message the user // submitted through the UI while the model was running. Though the UI // may support this, the model might not. - let pending_input = sess.get_pending_input().into_iter().map(ResponseItem::from); - net_new_turn_input.extend(pending_input); - - // Persist only the net-new items of this turn to the rollout. - sess.record_rollout_items(&net_new_turn_input).await; + let pending_input = sess + .get_pending_input() + .into_iter() + .map(ResponseItem::from) + .collect::>(); + sess.record_conversation_items(&pending_input).await; // Construct the input that we will send to the model. When using the // Chat completions API (or ZDR clients), the model needs the full @@ -796,20 +826,24 @@ async fn run_task(sess: Arc, sub_id: String, input: Vec) { // represents an append-only log without duplicates. let turn_input: Vec = if let Some(transcript) = sess.state.lock().unwrap().zdr_transcript.as_mut() { - // If we are using Chat/ZDR, we need to send the transcript with every turn. - - // 1. Build up the conversation history for the next turn. - let full_transcript = [transcript.contents(), net_new_turn_input.clone()].concat(); - - // 2. Update the in-memory transcript so that future turns - // include these items as part of the history. - transcript.record_items(&net_new_turn_input); - - // Note that `transcript.record_items()` does some filtering - // such that `full_transcript` may include items that were - // excluded from `transcript`. - full_transcript + // If we are using Chat/ZDR, we need to send the transcript with + // every turn. By induction, `transcript` already contains: + // - The `input` that kicked off this task. + // - Each `ResponseItem` that was recorded in the previous turn. + // - Each response to a `ResponseItem` (in practice, the only + // response type we seem to have is `FunctionCallOutput`). + // + // The only thing the `transcript` does not contain is the + // `pending_input` that was injected while the model was + // running. We need to add that to the conversation history + // so that the model can see it in the next turn. + [transcript.contents(), pending_input].concat() } else { + // In practice, net_new_turn_input should contain only: + // - User messages + // - Outputs for function calls requested by the model + net_new_turn_input.extend(pending_input); + // Responses API path – we can just send the new items and // record the same. net_new_turn_input @@ -830,29 +864,86 @@ async fn run_task(sess: Arc, sub_id: String, input: Vec) { .collect(); match run_turn(&sess, sub_id.clone(), turn_input).await { Ok(turn_output) => { - let (items, responses): (Vec<_>, Vec<_>) = turn_output - .into_iter() - .map(|p| (p.item, p.response)) - .unzip(); - let responses = responses - .into_iter() - .flatten() - .collect::>(); + let mut items_to_record_in_conversation_history = Vec::::new(); + let mut responses = Vec::::new(); + for processed_response_item in turn_output { + let ProcessedResponseItem { item, response } = processed_response_item; + match (&item, &response) { + (ResponseItem::Message { role, .. }, None) if role == "assistant" => { + // If the model returned a message, we need to record it. + items_to_record_in_conversation_history.push(item); + } + ( + ResponseItem::LocalShellCall { .. }, + Some(ResponseInputItem::FunctionCallOutput { call_id, output }), + ) => { + items_to_record_in_conversation_history.push(item); + items_to_record_in_conversation_history.push( + ResponseItem::FunctionCallOutput { + call_id: call_id.clone(), + output: output.clone(), + }, + ); + } + ( + ResponseItem::FunctionCall { .. }, + Some(ResponseInputItem::FunctionCallOutput { call_id, output }), + ) => { + items_to_record_in_conversation_history.push(item); + items_to_record_in_conversation_history.push( + ResponseItem::FunctionCallOutput { + call_id: call_id.clone(), + output: output.clone(), + }, + ); + } + ( + ResponseItem::FunctionCall { .. }, + Some(ResponseInputItem::McpToolCallOutput { call_id, result }), + ) => { + items_to_record_in_conversation_history.push(item); + let (content, success): (String, Option) = match result { + Ok(CallToolResult { content, is_error }) => { + match serde_json::to_string(content) { + Ok(content) => (content, *is_error), + Err(e) => { + warn!("Failed to serialize MCP tool call output: {e}"); + (e.to_string(), Some(true)) + } + } + } + Err(e) => (e.clone(), Some(true)), + }; + items_to_record_in_conversation_history.push( + ResponseItem::FunctionCallOutput { + call_id: call_id.clone(), + output: FunctionCallOutputPayload { content, success }, + }, + ); + } + (ResponseItem::Reasoning { .. }, None) => { + // Omit from conversation history. + } + _ => { + warn!("Unexpected response item: {item:?} with response: {response:?}"); + } + }; + if let Some(response) = response { + responses.push(response); + } + } // Only attempt to take the lock if there is something to record. - if !items.is_empty() { - // First persist model-generated output to the rollout file – this only borrows. - sess.record_rollout_items(&items).await; - - // For ZDR we also need to keep a transcript clone. - if let Some(transcript) = sess.state.lock().unwrap().zdr_transcript.as_mut() { - transcript.record_items(&items); - } + if !items_to_record_in_conversation_history.is_empty() { + sess.record_conversation_items(&items_to_record_in_conversation_history) + .await; } if responses.is_empty() { debug!("Turn completed"); - last_agent_message = get_last_assistant_message_from_turn(&items); + last_agent_message = get_last_assistant_message_from_turn( + &items_to_record_in_conversation_history, + ); sess.maybe_notify(UserNotification::AgentTurnComplete { turn_id: sub_id.clone(), input_messages: turn_input_messages, @@ -861,7 +952,7 @@ async fn run_task(sess: Arc, sub_id: String, input: Vec) { break; } - pending_response_input = responses; + input_for_next_turn = responses; } Err(e) => { info!("Turn error: {e:#}"); From 0f3cc8f8420e1014e14beb0acf970e51aea0b3bc Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Mon, 2 Jun 2025 16:01:34 -0700 Subject: [PATCH 27/41] feat: make reasoning effort/summaries configurable (#1199) Previous to this PR, we always set `reasoning` when making a request using the Responses API: https://github.com/openai/codex/blob/d7245cbbc9d8ff5446da45e5951761103492476d/codex-rs/core/src/client.rs#L108-L111 Though if you tried to use the Rust CLI with `--model gpt-4.1`, this would fail with: ```shell "Unsupported parameter: 'reasoning.effort' is not supported with this model." ``` We take a cue from the TypeScript CLI, which does a check on the model name: https://github.com/openai/codex/blob/d7245cbbc9d8ff5446da45e5951761103492476d/codex-cli/src/utils/agent/agent-loop.ts#L786-L789 This PR does a similar check, though also adds support for the following config options: ``` model_reasoning_effort = "low" | "medium" | "high" | "none" model_reasoning_summary = "auto" | "concise" | "detailed" | "none" ``` This way, if you have a model whose name happens to start with `"o"` (or `"codex"`?), you can set these to `"none"` to explicitly disable reasoning, if necessary. (That said, it seems unlikely anyone would use the Responses API with non-OpenAI models, but we provide an escape hatch, anyway.) This PR also updates both the TUI and `codex exec` to show `reasoning effort` and `reasoning summaries` in the header. --- codex-rs/Cargo.lock | 2 + codex-rs/config.md | 28 ++++++++++ codex-rs/core/Cargo.toml | 2 + codex-rs/core/src/client.rs | 26 ++++++--- codex-rs/core/src/client_common.rs | 84 +++++++++++++++++++++++++--- codex-rs/core/src/codex.rs | 11 +++- codex-rs/core/src/config.rs | 21 +++++++ codex-rs/core/src/config_types.rs | 30 ++++++++++ codex-rs/core/src/lib.rs | 2 + codex-rs/core/src/protocol.rs | 6 ++ codex-rs/exec/src/event_processor.rs | 18 +++++- codex-rs/tui/src/history_cell.rs | 16 +++++- 12 files changed, 226 insertions(+), 20 deletions(-) diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 97a90c1520..5f50faf9d0 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -635,6 +635,8 @@ dependencies = [ "seccompiler", "serde", "serde_json", + "strum 0.27.1", + "strum_macros 0.27.1", "tempfile", "thiserror 2.0.12", "time", diff --git a/codex-rs/config.md b/codex-rs/config.md index 416eeb4144..ffa735ff21 100644 --- a/codex-rs/config.md +++ b/codex-rs/config.md @@ -142,6 +142,34 @@ Users can specify config values at multiple levels. Order of precedence is as fo 3. as an entry in `config.toml`, e.g., `model = "o3"` 4. the default value that comes with Codex CLI (i.e., Codex CLI defaults to `codex-mini-latest`) +## model_reasoning_effort + +If the model name starts with `"o"` (as in `"o3"` or `"o4-mini"`) or `"codex"`, reasoning is enabled by default when using the Responses API. As explained in the [OpenAI Platform documentation](https://platform.openai.com/docs/guides/reasoning?api-mode=responses#get-started-with-reasoning), this can be set to: + +- `"low"` +- `"medium"` (default) +- `"high"` + +To disable reasoning, set `model_reasoning_effort` to `"none"` in your config: + +```toml +model_reasoning_effort = "none" # disable reasoning +``` + +## model_reasoning_summary + +If the model name starts with `"o"` (as in `"o3"` or `"o4-mini"`) or `"codex"`, reasoning is enabled by default when using the Responses API. As explained in the [OpenAI Platform documentation](https://platform.openai.com/docs/guides/reasoning?api-mode=responses#reasoning-summaries), this can be set to: + +- `"auto"` (default) +- `"concise"` +- `"detailed"` + +To disable reasoning summaries, set `model_reasoning_summary` to `"none"` in your config: + +```toml +model_reasoning_summary = "none" # disable reasoning summaries +``` + ## sandbox_permissions List of permissions to grant to the sandbox that Codex uses to execute untrusted commands: diff --git a/codex-rs/core/Cargo.toml b/codex-rs/core/Cargo.toml index 4687294981..4739ef31ed 100644 --- a/codex-rs/core/Cargo.toml +++ b/codex-rs/core/Cargo.toml @@ -31,6 +31,8 @@ rand = "0.9" reqwest = { version = "0.12", features = ["json", "stream"] } serde = { version = "1", features = ["derive"] } serde_json = "1" +strum = "0.27.1" +strum_macros = "0.27.1" thiserror = "2.0.12" time = { version = "0.3", features = ["formatting", "local-offset", "macros"] } tokio = { version = "1", features = [ diff --git a/codex-rs/core/src/client.rs b/codex-rs/core/src/client.rs index 6eb20149a5..74992fd178 100644 --- a/codex-rs/core/src/client.rs +++ b/codex-rs/core/src/client.rs @@ -18,12 +18,13 @@ use tracing::warn; use crate::chat_completions::AggregateStreamExt; use crate::chat_completions::stream_chat_completions; -use crate::client_common::Payload; use crate::client_common::Prompt; -use crate::client_common::Reasoning; use crate::client_common::ResponseEvent; use crate::client_common::ResponseStream; -use crate::client_common::Summary; +use crate::client_common::ResponsesApiRequest; +use crate::client_common::create_reasoning_param_for_request; +use crate::config_types::ReasoningEffort as ReasoningEffortConfig; +use crate::config_types::ReasoningSummary as ReasoningSummaryConfig; use crate::error::CodexErr; use crate::error::EnvVarError; use crate::error::Result; @@ -41,14 +42,23 @@ pub struct ModelClient { model: String, client: reqwest::Client, provider: ModelProviderInfo, + effort: ReasoningEffortConfig, + summary: ReasoningSummaryConfig, } impl ModelClient { - pub fn new(model: impl ToString, provider: ModelProviderInfo) -> Self { + pub fn new( + model: impl ToString, + provider: ModelProviderInfo, + effort: ReasoningEffortConfig, + summary: ReasoningSummaryConfig, + ) -> Self { Self { model: model.to_string(), client: reqwest::Client::new(), provider, + effort, + summary, } } @@ -98,17 +108,15 @@ impl ModelClient { let full_instructions = prompt.get_full_instructions(); let tools_json = create_tools_json_for_responses_api(prompt, &self.model)?; - let payload = Payload { + let reasoning = create_reasoning_param_for_request(&self.model, self.effort, self.summary); + let payload = ResponsesApiRequest { model: &self.model, instructions: &full_instructions, input: &prompt.input, tools: &tools_json, tool_choice: "auto", parallel_tool_calls: false, - reasoning: Some(Reasoning { - effort: "high", - summary: Some(Summary::Auto), - }), + reasoning, previous_response_id: prompt.prev_id.clone(), store: prompt.store, stream: true, diff --git a/codex-rs/core/src/client_common.rs b/codex-rs/core/src/client_common.rs index 8eb8074b1e..c4c3874cb2 100644 --- a/codex-rs/core/src/client_common.rs +++ b/codex-rs/core/src/client_common.rs @@ -1,3 +1,5 @@ +use crate::config_types::ReasoningEffort as ReasoningEffortConfig; +use crate::config_types::ReasoningSummary as ReasoningSummaryConfig; use crate::error::Result; use crate::models::ResponseItem; use futures::Stream; @@ -52,25 +54,59 @@ pub enum ResponseEvent { #[derive(Debug, Serialize)] pub(crate) struct Reasoning { - pub(crate) effort: &'static str, + pub(crate) effort: OpenAiReasoningEffort, #[serde(skip_serializing_if = "Option::is_none")] - pub(crate) summary: Option, + pub(crate) summary: Option, +} + +/// See https://platform.openai.com/docs/guides/reasoning?api-mode=responses#get-started-with-reasoning +#[derive(Debug, Serialize, Default, Clone, Copy)] +#[serde(rename_all = "lowercase")] +pub(crate) enum OpenAiReasoningEffort { + Low, + #[default] + Medium, + High, +} + +impl From for Option { + fn from(effort: ReasoningEffortConfig) -> Self { + match effort { + ReasoningEffortConfig::Low => Some(OpenAiReasoningEffort::Low), + ReasoningEffortConfig::Medium => Some(OpenAiReasoningEffort::Medium), + ReasoningEffortConfig::High => Some(OpenAiReasoningEffort::High), + ReasoningEffortConfig::None => None, + } + } } /// A summary of the reasoning performed by the model. This can be useful for /// debugging and understanding the model's reasoning process. -#[derive(Debug, Serialize)] +/// See https://platform.openai.com/docs/guides/reasoning?api-mode=responses#reasoning-summaries +#[derive(Debug, Serialize, Default, Clone, Copy)] #[serde(rename_all = "lowercase")] -pub(crate) enum Summary { +pub(crate) enum OpenAiReasoningSummary { + #[default] Auto, - #[allow(dead_code)] // Will go away once this is configurable. Concise, - #[allow(dead_code)] // Will go away once this is configurable. Detailed, } +impl From for Option { + fn from(summary: ReasoningSummaryConfig) -> Self { + match summary { + ReasoningSummaryConfig::Auto => Some(OpenAiReasoningSummary::Auto), + ReasoningSummaryConfig::Concise => Some(OpenAiReasoningSummary::Concise), + ReasoningSummaryConfig::Detailed => Some(OpenAiReasoningSummary::Detailed), + ReasoningSummaryConfig::None => None, + } + } +} + +/// Request object that is serialized as JSON and POST'ed when using the +/// Responses API. #[derive(Debug, Serialize)] -pub(crate) struct Payload<'a> { +pub(crate) struct ResponsesApiRequest<'a> { pub(crate) model: &'a str, pub(crate) instructions: &'a str, // TODO(mbolin): ResponseItem::Other should not be serialized. Currently, @@ -88,6 +124,40 @@ pub(crate) struct Payload<'a> { pub(crate) stream: bool, } +pub(crate) fn create_reasoning_param_for_request( + model: &str, + effort: ReasoningEffortConfig, + summary: ReasoningSummaryConfig, +) -> Option { + let effort: Option = effort.into(); + let effort = effort?; + + if model_supports_reasoning_summaries(model) { + Some(Reasoning { + effort, + summary: summary.into(), + }) + } else { + None + } +} + +pub fn model_supports_reasoning_summaries(model: &str) -> bool { + // Currently, we hardcode this rule to decide whether enable reasoning. + // We expect reasoning to apply only to OpenAI models, but we do not want + // users to have to mess with their config to disable reasoning for models + // that do not support it, such as `gpt-4.1`. + // + // Though if a user is using Codex with non-OpenAI models that, say, happen + // to start with "o", then they can set `model_reasoning_effort = "none` in + // config.toml to disable reasoning. + // + // Ultimately, this should also be configurable in config.toml, but we + // need to have defaults that "just work." Perhaps we could have a + // "reasoning models pattern" as part of ModelProviderInfo? + model.starts_with("o") || model.starts_with("codex") +} + pub(crate) struct ResponseStream { pub(crate) rx_event: mpsc::Receiver>, } diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index 01ff459f65..0a03fe60aa 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -108,6 +108,8 @@ impl Codex { let configure_session = Op::ConfigureSession { provider: config.model_provider.clone(), model: config.model.clone(), + model_reasoning_effort: config.model_reasoning_effort, + model_reasoning_summary: config.model_reasoning_summary, instructions, approval_policy: config.approval_policy, sandbox_policy: config.sandbox_policy.clone(), @@ -554,6 +556,8 @@ async fn submission_loop( Op::ConfigureSession { provider, model, + model_reasoning_effort, + model_reasoning_summary, instructions, approval_policy, sandbox_policy, @@ -575,7 +579,12 @@ async fn submission_loop( return; } - let client = ModelClient::new(model.clone(), provider.clone()); + let client = ModelClient::new( + model.clone(), + provider.clone(), + model_reasoning_effort, + model_reasoning_summary, + ); // abort any current running session and clone its state let retain_zdr_transcript = diff --git a/codex-rs/core/src/config.rs b/codex-rs/core/src/config.rs index d948ddb916..74798129ba 100644 --- a/codex-rs/core/src/config.rs +++ b/codex-rs/core/src/config.rs @@ -1,6 +1,8 @@ use crate::config_profile::ConfigProfile; use crate::config_types::History; use crate::config_types::McpServerConfig; +use crate::config_types::ReasoningEffort; +use crate::config_types::ReasoningSummary; use crate::config_types::ShellEnvironmentPolicy; use crate::config_types::ShellEnvironmentPolicyToml; use crate::config_types::Tui; @@ -112,6 +114,14 @@ pub struct Config { /// /// When this program is invoked, arg0 will be set to `codex-linux-sandbox`. pub codex_linux_sandbox_exe: Option, + + /// If not "none", the value to use for `reasoning.effort` when making a + /// request using the Responses API. + pub model_reasoning_effort: ReasoningEffort, + + /// If not "none", the value to use for `reasoning.summary` when making a + /// request using the Responses API. + pub model_reasoning_summary: ReasoningSummary, } impl Config { @@ -281,6 +291,9 @@ pub struct ConfigToml { /// When set to `true`, `AgentReasoning` events will be hidden from the /// UI/output. Defaults to `false`. pub hide_agent_reasoning: Option, + + pub model_reasoning_effort: Option, + pub model_reasoning_summary: Option, } fn deserialize_sandbox_permissions<'de, D>( @@ -444,6 +457,8 @@ impl Config { codex_linux_sandbox_exe, hide_agent_reasoning: cfg.hide_agent_reasoning.unwrap_or(false), + model_reasoning_effort: cfg.model_reasoning_effort.unwrap_or_default(), + model_reasoning_summary: cfg.model_reasoning_summary.unwrap_or_default(), }; Ok(config) } @@ -786,6 +801,8 @@ disable_response_storage = true tui: Tui::default(), codex_linux_sandbox_exe: None, hide_agent_reasoning: false, + model_reasoning_effort: ReasoningEffort::default(), + model_reasoning_summary: ReasoningSummary::default(), }, o3_profile_config ); @@ -826,6 +843,8 @@ disable_response_storage = true tui: Tui::default(), codex_linux_sandbox_exe: None, hide_agent_reasoning: false, + model_reasoning_effort: ReasoningEffort::default(), + model_reasoning_summary: ReasoningSummary::default(), }; assert_eq!(expected_gpt3_profile_config, gpt3_profile_config); @@ -881,6 +900,8 @@ disable_response_storage = true tui: Tui::default(), codex_linux_sandbox_exe: None, hide_agent_reasoning: false, + model_reasoning_effort: ReasoningEffort::default(), + model_reasoning_summary: ReasoningSummary::default(), }; assert_eq!(expected_zdr_profile_config, zdr_profile_config); diff --git a/codex-rs/core/src/config_types.rs b/codex-rs/core/src/config_types.rs index d89b09f267..a7152d1462 100644 --- a/codex-rs/core/src/config_types.rs +++ b/codex-rs/core/src/config_types.rs @@ -4,9 +4,11 @@ // definitions that do not contain business logic. use std::collections::HashMap; +use strum_macros::Display; use wildmatch::WildMatchPattern; use serde::Deserialize; +use serde::Serialize; #[derive(Deserialize, Debug, Clone, PartialEq)] pub struct McpServerConfig { @@ -175,3 +177,31 @@ impl From for ShellEnvironmentPolicy { } } } + +/// See https://platform.openai.com/docs/guides/reasoning?api-mode=responses#get-started-with-reasoning +#[derive(Debug, Serialize, Deserialize, Default, Clone, Copy, PartialEq, Eq, Display)] +#[serde(rename_all = "lowercase")] +#[strum(serialize_all = "lowercase")] +pub enum ReasoningEffort { + Low, + #[default] + Medium, + High, + /// Option to disable reasoning. + None, +} + +/// A summary of the reasoning performed by the model. This can be useful for +/// debugging and understanding the model's reasoning process. +/// See https://platform.openai.com/docs/guides/reasoning?api-mode=responses#reasoning-summaries +#[derive(Debug, Serialize, Deserialize, Default, Clone, Copy, PartialEq, Eq, Display)] +#[serde(rename_all = "lowercase")] +#[strum(serialize_all = "lowercase")] +pub enum ReasoningSummary { + #[default] + Auto, + Concise, + Detailed, + /// Option to disable reasoning summaries. + None, +} diff --git a/codex-rs/core/src/lib.rs b/codex-rs/core/src/lib.rs index 77941a9a51..1dcf67bd1c 100644 --- a/codex-rs/core/src/lib.rs +++ b/codex-rs/core/src/lib.rs @@ -34,3 +34,5 @@ mod rollout; mod safety; mod user_notification; pub mod util; + +pub use client_common::model_supports_reasoning_summaries; diff --git a/codex-rs/core/src/protocol.rs b/codex-rs/core/src/protocol.rs index fc18f1d821..737acc7732 100644 --- a/codex-rs/core/src/protocol.rs +++ b/codex-rs/core/src/protocol.rs @@ -12,6 +12,8 @@ use serde::Deserialize; use serde::Serialize; use uuid::Uuid; +use crate::config_types::ReasoningEffort as ReasoningEffortConfig; +use crate::config_types::ReasoningSummary as ReasoningSummaryConfig; use crate::message_history::HistoryEntry; use crate::model_provider_info::ModelProviderInfo; @@ -37,6 +39,10 @@ pub enum Op { /// If not specified, server will use its default model. model: String, + + model_reasoning_effort: ReasoningEffortConfig, + model_reasoning_summary: ReasoningSummaryConfig, + /// Model instructions instructions: Option, /// When to escalate for approval for execution diff --git a/codex-rs/exec/src/event_processor.rs b/codex-rs/exec/src/event_processor.rs index 5462736b5f..4cbbd25f0b 100644 --- a/codex-rs/exec/src/event_processor.rs +++ b/codex-rs/exec/src/event_processor.rs @@ -1,5 +1,7 @@ use codex_common::elapsed::format_elapsed; +use codex_core::WireApi; use codex_core::config::Config; +use codex_core::model_supports_reasoning_summaries; use codex_core::protocol::AgentMessageEvent; use codex_core::protocol::BackgroundEventEvent; use codex_core::protocol::ErrorEvent; @@ -127,16 +129,28 @@ impl EventProcessor { VERSION ); - let entries = vec![ + let mut entries = vec![ ("workdir", config.cwd.display().to_string()), ("model", config.model.clone()), ("provider", config.model_provider_id.clone()), ("approval", format!("{:?}", config.approval_policy)), ("sandbox", format!("{:?}", config.sandbox_policy)), ]; + if config.model_provider.wire_api == WireApi::Responses + && model_supports_reasoning_summaries(&config.model) + { + entries.push(( + "reasoning effort", + config.model_reasoning_effort.to_string(), + )); + entries.push(( + "reasoning summaries", + config.model_reasoning_summary.to_string(), + )); + } for (key, value) in entries { - println!("{} {}", format!("{key}: ").style(self.bold), value); + println!("{} {}", format!("{key}:").style(self.bold), value); } println!("--------"); diff --git a/codex-rs/tui/src/history_cell.rs b/codex-rs/tui/src/history_cell.rs index b41c8ac62b..a1fc672c6b 100644 --- a/codex-rs/tui/src/history_cell.rs +++ b/codex-rs/tui/src/history_cell.rs @@ -5,7 +5,9 @@ use crate::text_block::TextBlock; use base64::Engine; use codex_ansi_escape::ansi_escape_line; use codex_common::elapsed::format_duration; +use codex_core::WireApi; use codex_core::config::Config; +use codex_core::model_supports_reasoning_summaries; use codex_core::protocol::FileChange; use codex_core::protocol::SessionConfiguredEvent; use image::DynamicImage; @@ -147,13 +149,25 @@ impl HistoryCell { ]), ]; - let entries = vec![ + let mut entries = vec![ ("workdir", config.cwd.display().to_string()), ("model", config.model.clone()), ("provider", config.model_provider_id.clone()), ("approval", format!("{:?}", config.approval_policy)), ("sandbox", format!("{:?}", config.sandbox_policy)), ]; + if config.model_provider.wire_api == WireApi::Responses + && model_supports_reasoning_summaries(&config.model) + { + entries.push(( + "reasoning effort", + config.model_reasoning_effort.to_string(), + )); + entries.push(( + "reasoning summaries", + config.model_reasoning_summary.to_string(), + )); + } for (key, value) in entries { lines.push(Line::from(vec![format!("{key}: ").bold(), value.into()])); } From 5a5aa899143f9b9ef606692c401b010368b15bdb Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Mon, 2 Jun 2025 17:11:45 -0700 Subject: [PATCH 28/41] chore: replace regex with regex-lite, where appropriate (#1200) As explained on https://crates.io/crates/regex-lite, `regex-lite` is a lighter alternative to `regex` and seems to be sufficient for our purposes. --- codex-rs/Cargo.lock | 11 ++++++++--- codex-rs/apply-patch/Cargo.toml | 1 - codex-rs/execpolicy/Cargo.toml | 2 +- codex-rs/execpolicy/src/policy.rs | 6 +++--- codex-rs/execpolicy/src/policy_parser.rs | 8 ++++---- codex-rs/tui/Cargo.toml | 2 +- codex-rs/tui/src/citation_regex.rs | 2 +- codex-rs/tui/src/markdown.rs | 2 +- 8 files changed, 19 insertions(+), 15 deletions(-) diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 5f50faf9d0..694e11383f 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -567,7 +567,6 @@ version = "0.0.0" dependencies = [ "anyhow", "pretty_assertions", - "regex", "serde_json", "similar", "tempfile", @@ -682,7 +681,7 @@ dependencies = [ "log", "multimap", "path-absolutize", - "regex", + "regex-lite", "serde", "serde_json", "serde_with", @@ -757,7 +756,7 @@ dependencies = [ "pretty_assertions", "ratatui", "ratatui-image", - "regex", + "regex-lite", "serde_json", "shlex", "strum 0.27.1", @@ -3323,6 +3322,12 @@ dependencies = [ "regex-syntax 0.8.5", ] +[[package]] +name = "regex-lite" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53a49587ad06b26609c52e423de037e7f57f20d53535d66e08c695f347df952a" + [[package]] name = "regex-syntax" version = "0.6.29" diff --git a/codex-rs/apply-patch/Cargo.toml b/codex-rs/apply-patch/Cargo.toml index 66935b202f..1de09f86dd 100644 --- a/codex-rs/apply-patch/Cargo.toml +++ b/codex-rs/apply-patch/Cargo.toml @@ -12,7 +12,6 @@ workspace = true [dependencies] anyhow = "1" -regex = "1.11.1" serde_json = "1.0.110" similar = "2.7.0" thiserror = "2.0.12" diff --git a/codex-rs/execpolicy/Cargo.toml b/codex-rs/execpolicy/Cargo.toml index 9d9188c5b6..833df7ea3b 100644 --- a/codex-rs/execpolicy/Cargo.toml +++ b/codex-rs/execpolicy/Cargo.toml @@ -24,7 +24,7 @@ env_logger = "0.11.5" log = "0.4" multimap = "0.10.0" path-absolutize = "3.1.1" -regex = "1.11.1" +regex-lite = "0.1" serde = { version = "1.0.194", features = ["derive"] } serde_json = "1.0.110" serde_with = { version = "3", features = ["macros"] } diff --git a/codex-rs/execpolicy/src/policy.rs b/codex-rs/execpolicy/src/policy.rs index 5dd1355081..d1fe4ea896 100644 --- a/codex-rs/execpolicy/src/policy.rs +++ b/codex-rs/execpolicy/src/policy.rs @@ -1,6 +1,6 @@ use multimap::MultiMap; -use regex::Error as RegexError; -use regex::Regex; +use regex_lite::Error as RegexError; +use regex_lite::Regex; use crate::ExecCall; use crate::Forbidden; @@ -29,7 +29,7 @@ impl Policy { } else { let escaped_substrings = forbidden_substrings .iter() - .map(|s| regex::escape(s)) + .map(|s| regex_lite::escape(s)) .collect::>() .join("|"); Some(Regex::new(&format!("({escaped_substrings})"))?) diff --git a/codex-rs/execpolicy/src/policy_parser.rs b/codex-rs/execpolicy/src/policy_parser.rs index 92ed0bdc70..0290619d09 100644 --- a/codex-rs/execpolicy/src/policy_parser.rs +++ b/codex-rs/execpolicy/src/policy_parser.rs @@ -7,7 +7,7 @@ use crate::arg_matcher::ArgMatcher; use crate::opt::OptMeta; use log::info; use multimap::MultiMap; -use regex::Regex; +use regex_lite::Regex; use starlark::any::ProvidesStaticType; use starlark::environment::GlobalsBuilder; use starlark::environment::LibraryExtension; @@ -73,7 +73,7 @@ impl PolicyParser { #[derive(Debug)] pub struct ForbiddenProgramRegex { - pub regex: regex::Regex, + pub regex: regex_lite::Regex, pub reason: String, } @@ -93,7 +93,7 @@ impl PolicyBuilder { } } - fn build(self) -> Result { + fn build(self) -> Result { let programs = self.programs.into_inner(); let forbidden_program_regexes = self.forbidden_program_regexes.into_inner(); let forbidden_substrings = self.forbidden_substrings.into_inner(); @@ -207,7 +207,7 @@ fn policy_builtins(builder: &mut GlobalsBuilder) { .unwrap() .downcast_ref::() .unwrap(); - let compiled_regex = regex::Regex::new(®ex)?; + let compiled_regex = regex_lite::Regex::new(®ex)?; policy_builder.add_forbidden_program_regex(compiled_regex, reason); Ok(NoneType) } diff --git a/codex-rs/tui/Cargo.toml b/codex-rs/tui/Cargo.toml index 5886ce69dc..235f5f0c7a 100644 --- a/codex-rs/tui/Cargo.toml +++ b/codex-rs/tui/Cargo.toml @@ -33,7 +33,7 @@ ratatui = { version = "0.29.0", features = [ "unstable-rendered-line-info", ] } ratatui-image = "8.0.0" -regex = "1" +regex-lite = "0.1" serde_json = "1" shlex = "1.3.0" strum = "0.27.1" diff --git a/codex-rs/tui/src/citation_regex.rs b/codex-rs/tui/src/citation_regex.rs index 7cda1ef11f..e5355ec2b8 100644 --- a/codex-rs/tui/src/citation_regex.rs +++ b/codex-rs/tui/src/citation_regex.rs @@ -1,6 +1,6 @@ #![allow(clippy::expect_used)] -use regex::Regex; +use regex_lite::Regex; // This is defined in its own file so we can limit the scope of // `allow(clippy::expect_used)` because we cannot scope it to the `lazy_static!` diff --git a/codex-rs/tui/src/markdown.rs b/codex-rs/tui/src/markdown.rs index a56ce7749e..ab20138298 100644 --- a/codex-rs/tui/src/markdown.rs +++ b/codex-rs/tui/src/markdown.rs @@ -71,7 +71,7 @@ fn rewrite_file_citations<'a>( None => return Cow::Borrowed(src), }; - CITATION_REGEX.replace_all(src, |caps: ®ex::Captures<'_>| { + CITATION_REGEX.replace_all(src, |caps: ®ex_lite::Captures<'_>| { let file = &caps[1]; let start_line = &caps[2]; From 6fcc528a437a8fcf6aa070d3d1ac986c3d1a51bf Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Tue, 3 Jun 2025 09:06:38 -0700 Subject: [PATCH 29/41] fix: provide tolerance for apply_patch tool (#993) As explained in detail in the doc comment for `ParseMode::Lenient`, we have observed that GPT-4.1 does not always generate a valid invocation of `apply_patch`. Fortunately, the error is predictable, so we introduce some new logic to the `codex-apply-patch` crate to recover from this error. Because we would like to avoid this becoming a de facto standard (as it would be incompatible if `apply_patch` were provided as an actual executable, unless we also introduced the lenient behavior in the executable, as well), we require passing `ParseMode::Lenient` to `parse_patch_text()` to make it clear that the caller is opting into supporting this special case. Note the analogous change to the TypeScript CLI was https://github.com/openai/codex/pull/930. In addition to changing the accepted input to `apply_patch`, it also introduced additional instructions for the model, which we include in this PR. Note that `apply-patch` does not depend on either `regex` or `regex-lite`, so some of the checks are slightly more verbose to avoid introducing this dependency. That said, this PR does not leverage the existing `extract_heredoc_body_from_apply_patch_command()`, which depends on `tree-sitter` and `tree-sitter-bash`: https://github.com/openai/codex/blob/5a5aa899143f9b9ef606692c401b010368b15bdb/codex-rs/apply-patch/src/lib.rs#L191-L246 though perhaps it should. --- .../apply_patch_tool_instructions.md | 40 +++ codex-rs/apply-patch/src/lib.rs | 3 + codex-rs/apply-patch/src/parser.rs | 244 ++++++++++++++++-- codex-rs/core/src/chat_completions.rs | 2 +- codex-rs/core/src/client.rs | 2 +- codex-rs/core/src/client_common.rs | 25 +- 6 files changed, 281 insertions(+), 35 deletions(-) create mode 100644 codex-rs/apply-patch/apply_patch_tool_instructions.md diff --git a/codex-rs/apply-patch/apply_patch_tool_instructions.md b/codex-rs/apply-patch/apply_patch_tool_instructions.md new file mode 100644 index 0000000000..3c51d9cfbf --- /dev/null +++ b/codex-rs/apply-patch/apply_patch_tool_instructions.md @@ -0,0 +1,40 @@ +To edit files, ALWAYS use the `shell` tool with `apply_patch` CLI. `apply_patch` effectively allows you to execute a diff/patch against a file, but the format of the diff specification is unique to this task, so pay careful attention to these instructions. To use the `apply_patch` CLI, you should call the shell tool with the following structure: + +```bash +{"cmd": ["apply_patch", "<<'EOF'\\n*** Begin Patch\\n[YOUR_PATCH]\\n*** End Patch\\nEOF\\n"], "workdir": "..."} +``` + +Where [YOUR_PATCH] is the actual content of your patch, specified in the following V4A diff format. + +*** [ACTION] File: [path/to/file] -> ACTION can be one of Add, Update, or Delete. +For each snippet of code that needs to be changed, repeat the following: +[context_before] -> See below for further instructions on context. +- [old_code] -> Precede the old code with a minus sign. ++ [new_code] -> Precede the new, replacement code with a plus sign. +[context_after] -> See below for further instructions on context. + +For instructions on [context_before] and [context_after]: +- By default, show 3 lines of code immediately above and 3 lines immediately below each change. If a change is within 3 lines of a previous change, do NOT duplicate the first change’s [context_after] lines in the second change’s [context_before] lines. +- If 3 lines of context is insufficient to uniquely identify the snippet of code within the file, use the @@ operator to indicate the class or function to which the snippet belongs. For instance, we might have: +@@ class BaseClass +[3 lines of pre-context] +- [old_code] ++ [new_code] +[3 lines of post-context] + +- If a code block is repeated so many times in a class or function such that even a single `@@` statement and 3 lines of context cannot uniquely identify the snippet of code, you can use multiple `@@` statements to jump to the right context. For instance: + +@@ class BaseClass +@@ def method(): +[3 lines of pre-context] +- [old_code] ++ [new_code] +[3 lines of post-context] + +Note, then, that we do not use line numbers in this diff format, as the context is enough to uniquely identify code. An example of a message that you might pass as "input" to this function, in order to apply a patch, is shown below. + +```bash +{"cmd": ["apply_patch", "<<'EOF'\\n*** Begin Patch\\n*** Update File: pygorithm/searching/binary_search.py\\n@@ class BaseClass\\n@@ def search():\\n- pass\\n+ raise NotImplementedError()\\n@@ class Subclass\\n@@ def search():\\n- pass\\n+ raise NotImplementedError()\\n*** End Patch\\nEOF\\n"], "workdir": "..."} +``` + +File references can only be relative, NEVER ABSOLUTE. After the apply_patch command is run, it will always say "Done!", regardless of whether the patch was successfully applied or not. However, you can determine if there are issue and errors by looking at any warnings or logging lines printed BEFORE the "Done!" is output. diff --git a/codex-rs/apply-patch/src/lib.rs b/codex-rs/apply-patch/src/lib.rs index fcbc97b4f6..5a5290bffa 100644 --- a/codex-rs/apply-patch/src/lib.rs +++ b/codex-rs/apply-patch/src/lib.rs @@ -19,6 +19,9 @@ use tree_sitter::LanguageError; use tree_sitter::Parser; use tree_sitter_bash::LANGUAGE as BASH; +/// Detailed instructions for gpt-4.1 on how to use the `apply_patch` tool. +pub const APPLY_PATCH_TOOL_INSTRUCTIONS: &str = include_str!("../apply_patch_tool_instructions.md"); + #[derive(Debug, Error, PartialEq)] pub enum ApplyPatchError { #[error(transparent)] diff --git a/codex-rs/apply-patch/src/parser.rs b/codex-rs/apply-patch/src/parser.rs index 391255defa..d07691a49d 100644 --- a/codex-rs/apply-patch/src/parser.rs +++ b/codex-rs/apply-patch/src/parser.rs @@ -37,7 +37,15 @@ const EOF_MARKER: &str = "*** End of File"; const CHANGE_CONTEXT_MARKER: &str = "@@ "; const EMPTY_CHANGE_CONTEXT_MARKER: &str = "@@"; -#[derive(Debug, PartialEq, Error)] +/// Currently, the only OpenAI model that knowingly requires lenient parsing is +/// gpt-4.1. While we could try to require everyone to pass in a strictness +/// param when invoking apply_patch, it is a pain to thread it through all of +/// the call sites, so we resign ourselves allowing lenient parsing for all +/// models. See [`ParseMode::Lenient`] for details on the exceptions we make for +/// gpt-4.1. +const PARSE_IN_STRICT_MODE: bool = false; + +#[derive(Debug, PartialEq, Error, Clone)] pub enum ParseError { #[error("invalid patch: {0}")] InvalidPatchError(String), @@ -46,7 +54,7 @@ pub enum ParseError { } use ParseError::*; -#[derive(Debug, PartialEq)] +#[derive(Debug, PartialEq, Clone)] #[allow(clippy::enum_variant_names)] pub enum Hunk { AddFile { @@ -78,7 +86,7 @@ impl Hunk { use Hunk::*; -#[derive(Debug, PartialEq)] +#[derive(Debug, PartialEq, Clone)] pub struct UpdateFileChunk { /// A single line of context used to narrow down the position of the chunk /// (this is usually a class, method, or function definition.) @@ -95,19 +103,68 @@ pub struct UpdateFileChunk { } pub fn parse_patch(patch: &str) -> Result, ParseError> { + let mode = if PARSE_IN_STRICT_MODE { + ParseMode::Strict + } else { + ParseMode::Lenient + }; + parse_patch_text(patch, mode) +} + +enum ParseMode { + /// Parse the patch text argument as is. + Strict, + + /// GPT-4.1 is known to formulate the `command` array for the `local_shell` + /// tool call for `apply_patch` call using something like the following: + /// + /// ```json + /// [ + /// "apply_patch", + /// "<<'EOF'\n*** Begin Patch\n*** Update File: README.md\n@@...\n*** End Patch\nEOF\n", + /// ] + /// ``` + /// + /// This is a problem because `local_shell` is a bit of a misnomer: the + /// `command` is not invoked by passing the arguments to a shell like Bash, + /// but are invoked using something akin to `execvpe(3)`. + /// + /// This is significant in this case because where a shell would interpret + /// `<<'EOF'...` as a heredoc and pass the contents via stdin (which is + /// fine, as `apply_patch` is specified to read from stdin if no argument is + /// passed), `execvpe(3)` interprets the heredoc as a literal string. To get + /// the `local_shell` tool to run a command the way shell would, the + /// `command` array must be something like: + /// + /// ```json + /// [ + /// "bash", + /// "-lc", + /// "apply_patch <<'EOF'\n*** Begin Patch\n*** Update File: README.md\n@@...\n*** End Patch\nEOF\n", + /// ] + /// ``` + /// + /// In lenient mode, we check if the argument to `apply_patch` starts with + /// `<<'EOF'` and ends with `EOF\n`. If so, we strip off these markers, + /// trim() the result, and treat what is left as the patch text. + Lenient, +} + +fn parse_patch_text(patch: &str, mode: ParseMode) -> Result, ParseError> { let lines: Vec<&str> = patch.trim().lines().collect(); - if lines.is_empty() || lines[0] != BEGIN_PATCH_MARKER { - return Err(InvalidPatchError(String::from( - "The first line of the patch must be '*** Begin Patch'", - ))); - } - let last_line_index = lines.len() - 1; - if lines[last_line_index] != END_PATCH_MARKER { - return Err(InvalidPatchError(String::from( - "The last line of the patch must be '*** End Patch'", - ))); - } + let lines: &[&str] = match check_patch_boundaries_strict(&lines) { + Ok(()) => &lines, + Err(e) => match mode { + ParseMode::Strict => { + return Err(e); + } + ParseMode::Lenient => check_patch_boundaries_lenient(&lines, e)?, + }, + }; + let mut hunks: Vec = Vec::new(); + // The above checks ensure that lines.len() >= 2. + let last_line_index = lines.len().saturating_sub(1); let mut remaining_lines = &lines[1..last_line_index]; let mut line_number = 2; while !remaining_lines.is_empty() { @@ -119,6 +176,64 @@ pub fn parse_patch(patch: &str) -> Result, ParseError> { Ok(hunks) } +/// Checks the start and end lines of the patch text for `apply_patch`, +/// returning an error if they do not match the expected markers. +fn check_patch_boundaries_strict(lines: &[&str]) -> Result<(), ParseError> { + let (first_line, last_line) = match lines { + [] => (None, None), + [first] => (Some(first), Some(first)), + [first, .., last] => (Some(first), Some(last)), + }; + check_start_and_end_lines_strict(first_line, last_line) +} + +/// If we are in lenient mode, we check if the first line starts with `<( + original_lines: &'a [&'a str], + original_parse_error: ParseError, +) -> Result<&'a [&'a str], ParseError> { + match original_lines { + [first, .., last] => { + if (first == &"<= 4 + { + let inner_lines = &original_lines[1..original_lines.len() - 1]; + match check_patch_boundaries_strict(inner_lines) { + Ok(()) => Ok(inner_lines), + Err(e) => Err(e), + } + } else { + Err(original_parse_error) + } + } + _ => Err(original_parse_error), + } +} + +fn check_start_and_end_lines_strict( + first_line: Option<&&str>, + last_line: Option<&&str>, +) -> Result<(), ParseError> { + match (first_line, last_line) { + (Some(&first), Some(&last)) if first == BEGIN_PATCH_MARKER && last == END_PATCH_MARKER => { + Ok(()) + } + (Some(&first), _) if first != BEGIN_PATCH_MARKER => Err(InvalidPatchError(String::from( + "The first line of the patch must be '*** Begin Patch'", + ))), + _ => Err(InvalidPatchError(String::from( + "The last line of the patch must be '*** End Patch'", + ))), + } +} + /// Attempts to parse a single hunk from the start of lines. /// Returns the parsed hunk and the number of lines parsed (or a ParseError). fn parse_one_hunk(lines: &[&str], line_number: usize) -> Result<(Hunk, usize), ParseError> { @@ -312,22 +427,23 @@ fn parse_update_file_chunk( #[test] fn test_parse_patch() { assert_eq!( - parse_patch("bad"), + parse_patch_text("bad", ParseMode::Strict), Err(InvalidPatchError( "The first line of the patch must be '*** Begin Patch'".to_string() )) ); assert_eq!( - parse_patch("*** Begin Patch\nbad"), + parse_patch_text("*** Begin Patch\nbad", ParseMode::Strict), Err(InvalidPatchError( "The last line of the patch must be '*** End Patch'".to_string() )) ); assert_eq!( - parse_patch( + parse_patch_text( "*** Begin Patch\n\ *** Update File: test.py\n\ - *** End Patch" + *** End Patch", + ParseMode::Strict ), Err(InvalidHunkError { message: "Update file hunk for path 'test.py' is empty".to_string(), @@ -335,14 +451,15 @@ fn test_parse_patch() { }) ); assert_eq!( - parse_patch( + parse_patch_text( "*** Begin Patch\n\ - *** End Patch" + *** End Patch", + ParseMode::Strict ), Ok(Vec::new()) ); assert_eq!( - parse_patch( + parse_patch_text( "*** Begin Patch\n\ *** Add File: path/add.py\n\ +abc\n\ @@ -353,7 +470,8 @@ fn test_parse_patch() { @@ def f():\n\ - pass\n\ + return 123\n\ - *** End Patch" + *** End Patch", + ParseMode::Strict ), Ok(vec![ AddFile { @@ -377,14 +495,15 @@ fn test_parse_patch() { ); // Update hunk followed by another hunk (Add File). assert_eq!( - parse_patch( + parse_patch_text( "*** Begin Patch\n\ *** Update File: file.py\n\ @@\n\ +line\n\ *** Add File: other.py\n\ +content\n\ - *** End Patch" + *** End Patch", + ParseMode::Strict ), Ok(vec![ UpdateFile { @@ -407,12 +526,13 @@ fn test_parse_patch() { // Update hunk without an explicit @@ header for the first chunk should parse. // Use a raw string to preserve the leading space diff marker on the context line. assert_eq!( - parse_patch( + parse_patch_text( r#"*** Begin Patch *** Update File: file2.py import foo +bar *** End Patch"#, + ParseMode::Strict ), Ok(vec![UpdateFile { path: PathBuf::from("file2.py"), @@ -427,6 +547,80 @@ fn test_parse_patch() { ); } +#[test] +fn test_parse_patch_lenient() { + let patch_text = r#"*** Begin Patch +*** Update File: file2.py + import foo ++bar +*** End Patch"#; + let expected_patch = vec![UpdateFile { + path: PathBuf::from("file2.py"), + move_path: None, + chunks: vec![UpdateFileChunk { + change_context: None, + old_lines: vec!["import foo".to_string()], + new_lines: vec!["import foo".to_string(), "bar".to_string()], + is_end_of_file: false, + }], + }]; + let expected_error = + InvalidPatchError("The first line of the patch must be '*** Begin Patch'".to_string()); + + let patch_text_in_heredoc = format!("<::new(); - let full_instructions = prompt.get_full_instructions(); + let full_instructions = prompt.get_full_instructions(model); messages.push(json!({"role": "system", "content": full_instructions})); for item in &prompt.input { diff --git a/codex-rs/core/src/client.rs b/codex-rs/core/src/client.rs index 74992fd178..aff838887a 100644 --- a/codex-rs/core/src/client.rs +++ b/codex-rs/core/src/client.rs @@ -106,7 +106,7 @@ impl ModelClient { return stream_from_fixture(path).await; } - let full_instructions = prompt.get_full_instructions(); + let full_instructions = prompt.get_full_instructions(&self.model); let tools_json = create_tools_json_for_responses_api(prompt, &self.model)?; let reasoning = create_reasoning_param_for_request(&self.model, self.effort, self.summary); let payload = ResponsesApiRequest { diff --git a/codex-rs/core/src/client_common.rs b/codex-rs/core/src/client_common.rs index c4c3874cb2..3692880d72 100644 --- a/codex-rs/core/src/client_common.rs +++ b/codex-rs/core/src/client_common.rs @@ -2,6 +2,7 @@ use crate::config_types::ReasoningEffort as ReasoningEffortConfig; use crate::config_types::ReasoningSummary as ReasoningSummaryConfig; use crate::error::Result; use crate::models::ResponseItem; +use codex_apply_patch::APPLY_PATCH_TOOL_INSTRUCTIONS; use futures::Stream; use serde::Serialize; use std::borrow::Cow; @@ -35,14 +36,22 @@ pub struct Prompt { } impl Prompt { - pub(crate) fn get_full_instructions(&self) -> Cow { - match &self.instructions { - Some(instructions) => { - let instructions = format!("{BASE_INSTRUCTIONS}\n{instructions}"); - Cow::Owned(instructions) - } - None => Cow::Borrowed(BASE_INSTRUCTIONS), - } + pub(crate) fn get_full_instructions(&self, model: &str) -> Cow { + [ + Some(Cow::Borrowed(BASE_INSTRUCTIONS)), + self.instructions.as_ref().map(|s| Cow::Owned(s.clone())), + if model.starts_with("gpt-4.1") { + Some(Cow::Borrowed(APPLY_PATCH_TOOL_INSTRUCTIONS)) + } else { + None + }, + ] + .iter() + .filter_map(|s| s.as_ref()) + .map(|cow| cow.as_ref()) + .collect::>() + .join("\n") + .into() } } From c6fcec55fe4085b4885bd99e40b46756f12511fd Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Tue, 3 Jun 2025 09:40:19 -0700 Subject: [PATCH 30/41] fix: always send full instructions when using the Responses API (#1207) This fixes a longstanding error in the Rust CLI where `codex.rs` contained an errant `is_first_turn` check that would exclude the user instructions for subsequent "turns" of a conversation when using the responses API (i.e., when `previous_response_id` existed). While here, renames `Prompt.instructions` to `Prompt.user_instructions` since we now have quite a few levels of instructions floating around. Also removed an unnecessary use of `clone()` in `Prompt.get_full_instructions()`. --- codex-rs/core/src/client_common.rs | 25 +++++++++---------------- codex-rs/core/src/codex.rs | 17 +++++------------ codex-rs/core/src/project_doc.rs | 16 ++++++++-------- 3 files changed, 22 insertions(+), 36 deletions(-) diff --git a/codex-rs/core/src/client_common.rs b/codex-rs/core/src/client_common.rs index 3692880d72..a2633475df 100644 --- a/codex-rs/core/src/client_common.rs +++ b/codex-rs/core/src/client_common.rs @@ -25,7 +25,7 @@ pub struct Prompt { pub prev_id: Option, /// Optional instructions from the user to amend to the built-in agent /// instructions. - pub instructions: Option, + pub user_instructions: Option, /// Whether to store response on server side (disable_response_storage = !store). pub store: bool, @@ -37,21 +37,14 @@ pub struct Prompt { impl Prompt { pub(crate) fn get_full_instructions(&self, model: &str) -> Cow { - [ - Some(Cow::Borrowed(BASE_INSTRUCTIONS)), - self.instructions.as_ref().map(|s| Cow::Owned(s.clone())), - if model.starts_with("gpt-4.1") { - Some(Cow::Borrowed(APPLY_PATCH_TOOL_INSTRUCTIONS)) - } else { - None - }, - ] - .iter() - .filter_map(|s| s.as_ref()) - .map(|cow| cow.as_ref()) - .collect::>() - .join("\n") - .into() + let mut sections: Vec<&str> = vec![BASE_INSTRUCTIONS]; + if let Some(ref user) = self.user_instructions { + sections.push(user); + } + if model.starts_with("gpt-4.1") { + sections.push(APPLY_PATCH_TOOL_INSTRUCTIONS); + } + Cow::Owned(sections.join("\n")) } } diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index 0a03fe60aa..2837dd032e 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -59,7 +59,7 @@ use crate::models::ReasoningItemReasoningSummary; use crate::models::ResponseInputItem; use crate::models::ResponseItem; use crate::models::ShellToolCallParams; -use crate::project_doc::create_full_instructions; +use crate::project_doc::get_user_instructions; use crate::protocol::AgentMessageEvent; use crate::protocol::AgentReasoningEvent; use crate::protocol::ApplyPatchApprovalRequestEvent; @@ -104,7 +104,7 @@ impl Codex { let (tx_sub, rx_sub) = async_channel::bounded(64); let (tx_event, rx_event) = async_channel::bounded(64); - let instructions = create_full_instructions(&config).await; + let instructions = get_user_instructions(&config).await; let configure_session = Op::ConfigureSession { provider: config.model_provider.clone(), model: config.model.clone(), @@ -990,9 +990,8 @@ async fn run_turn( input: Vec, ) -> CodexResult> { // Decide whether to use server-side storage (previous_response_id) or disable it - let (prev_id, store, is_first_turn) = { + let (prev_id, store) = { let state = sess.state.lock().unwrap(); - let is_first_turn = state.previous_response_id.is_none(); let store = state.zdr_transcript.is_none(); let prev_id = if store { state.previous_response_id.clone() @@ -1001,20 +1000,14 @@ async fn run_turn( // back, but trying to use it results in a 400. None }; - (prev_id, store, is_first_turn) - }; - - let instructions = if is_first_turn { - sess.instructions.clone() - } else { - None + (prev_id, store) }; let extra_tools = sess.mcp_connection_manager.list_all_tools(); let prompt = Prompt { input, prev_id, - instructions, + user_instructions: sess.instructions.clone(), store, extra_tools, }; diff --git a/codex-rs/core/src/project_doc.rs b/codex-rs/core/src/project_doc.rs index 1a4e90debc..ab9d46186f 100644 --- a/codex-rs/core/src/project_doc.rs +++ b/codex-rs/core/src/project_doc.rs @@ -25,7 +25,7 @@ const PROJECT_DOC_SEPARATOR: &str = "\n\n--- project-doc ---\n\n"; /// Combines `Config::instructions` and `AGENTS.md` (if present) into a single /// string of instructions. -pub(crate) async fn create_full_instructions(config: &Config) -> Option { +pub(crate) async fn get_user_instructions(config: &Config) -> Option { match find_project_doc(config).await { Ok(Some(project_doc)) => match &config.instructions { Some(original_instructions) => Some(format!( @@ -168,7 +168,7 @@ mod tests { async fn no_doc_file_returns_none() { let tmp = tempfile::tempdir().expect("tempdir"); - let res = create_full_instructions(&make_config(&tmp, 4096, None)).await; + let res = get_user_instructions(&make_config(&tmp, 4096, None)).await; assert!( res.is_none(), "Expected None when AGENTS.md is absent and no system instructions provided" @@ -182,7 +182,7 @@ mod tests { let tmp = tempfile::tempdir().expect("tempdir"); fs::write(tmp.path().join("AGENTS.md"), "hello world").unwrap(); - let res = create_full_instructions(&make_config(&tmp, 4096, None)) + let res = get_user_instructions(&make_config(&tmp, 4096, None)) .await .expect("doc expected"); @@ -201,7 +201,7 @@ mod tests { let huge = "A".repeat(LIMIT * 2); // 2 KiB fs::write(tmp.path().join("AGENTS.md"), &huge).unwrap(); - let res = create_full_instructions(&make_config(&tmp, LIMIT, None)) + let res = get_user_instructions(&make_config(&tmp, LIMIT, None)) .await .expect("doc expected"); @@ -233,7 +233,7 @@ mod tests { let mut cfg = make_config(&repo, 4096, None); cfg.cwd = nested; - let res = create_full_instructions(&cfg).await.expect("doc expected"); + let res = get_user_instructions(&cfg).await.expect("doc expected"); assert_eq!(res, "root level doc"); } @@ -243,7 +243,7 @@ mod tests { let tmp = tempfile::tempdir().expect("tempdir"); fs::write(tmp.path().join("AGENTS.md"), "something").unwrap(); - let res = create_full_instructions(&make_config(&tmp, 0, None)).await; + let res = get_user_instructions(&make_config(&tmp, 0, None)).await; assert!( res.is_none(), "With limit 0 the function should return None" @@ -259,7 +259,7 @@ mod tests { const INSTRUCTIONS: &str = "base instructions"; - let res = create_full_instructions(&make_config(&tmp, 4096, Some(INSTRUCTIONS))) + let res = get_user_instructions(&make_config(&tmp, 4096, Some(INSTRUCTIONS))) .await .expect("should produce a combined instruction string"); @@ -276,7 +276,7 @@ mod tests { const INSTRUCTIONS: &str = "some instructions"; - let res = create_full_instructions(&make_config(&tmp, 4096, Some(INSTRUCTIONS))).await; + let res = get_user_instructions(&make_config(&tmp, 4096, Some(INSTRUCTIONS))).await; assert_eq!(res, Some(INSTRUCTIONS.to_string())); } From a67a67f3258fc21e147b6786a143fe3e15e6d5ba Mon Sep 17 00:00:00 2001 From: Reilly Wood <163153147+rgwood-dd@users.noreply.github.com> Date: Tue, 3 Jun 2025 14:29:26 -0700 Subject: [PATCH 31/41] codex-rs: make tool calls prettier (#1211) This PR overhauls how active tool calls and completed tool calls are displayed: 1. More use of colour to indicate success/failure and distinguish between components like tool name+arguments 2. Previously, the entire `CallToolResult` was serialized to JSON and pretty-printed. Now, we extract each individual `CallToolResultContent` and print those 1. The previous solution was wasting space by unnecessarily showing details of the `CallToolResult` struct to users, without formatting the actual tool call results nicely 2. We're now able to show users more information from tool results in less space, with nicer formatting when tools return JSON results ### Before: Screenshot 2025-06-03 at 11 24 26 ### After: image ## Future Work 1. Integrate image tool result handling better. We should be able to display images even if they're not the first `CallToolResultContent` 2. Users should have some way to view the full version of truncated tool results 3. It would be nice to add some left padding for tool results, make it more clear that they are results. This is doable, just a little fiddly due to the way `first_visible_line` scrolling works 4. There's almost certainly a better way to format JSON than "all on 1 line with spaces to make Ratatui wrapping work". But I think that works OK for now. --- codex-rs/Cargo.lock | 1 + codex-rs/mcp-types/src/lib.rs | 1 + codex-rs/tui/Cargo.toml | 3 +- .../tui/src/conversation_history_widget.rs | 3 +- codex-rs/tui/src/history_cell.rs | 125 +++++--- codex-rs/tui/src/lib.rs | 1 + codex-rs/tui/src/text_formatting.rs | 268 ++++++++++++++++++ 7 files changed, 352 insertions(+), 50 deletions(-) create mode 100644 codex-rs/tui/src/text_formatting.rs diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 694e11383f..ae574ba303 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -768,6 +768,7 @@ dependencies = [ "tui-input", "tui-markdown", "tui-textarea", + "unicode-segmentation", "uuid", ] diff --git a/codex-rs/mcp-types/src/lib.rs b/codex-rs/mcp-types/src/lib.rs index afd6f4ad63..0ed518535f 100644 --- a/codex-rs/mcp-types/src/lib.rs +++ b/codex-rs/mcp-types/src/lib.rs @@ -1144,6 +1144,7 @@ pub enum ServerRequest { #[derive(Debug, Clone, PartialEq, Deserialize, Serialize)] #[serde(untagged)] +#[allow(clippy::large_enum_variant)] pub enum ServerResult { Result(Result), InitializeResult(InitializeResult), diff --git a/codex-rs/tui/Cargo.toml b/codex-rs/tui/Cargo.toml index 235f5f0c7a..ffc107e831 100644 --- a/codex-rs/tui/Cargo.toml +++ b/codex-rs/tui/Cargo.toml @@ -34,7 +34,7 @@ ratatui = { version = "0.29.0", features = [ ] } ratatui-image = "8.0.0" regex-lite = "0.1" -serde_json = "1" +serde_json = { version = "1", features = ["preserve_order"] } shlex = "1.3.0" strum = "0.27.1" strum_macros = "0.27.1" @@ -51,6 +51,7 @@ tracing-subscriber = { version = "0.3.19", features = ["env-filter"] } tui-input = "0.11.1" tui-markdown = "0.3.3" tui-textarea = "0.7.0" +unicode-segmentation = "1.12.0" uuid = "1" [dev-dependencies] diff --git a/codex-rs/tui/src/conversation_history_widget.rs b/codex-rs/tui/src/conversation_history_widget.rs index 9242e00389..a23e00d776 100644 --- a/codex-rs/tui/src/conversation_history_widget.rs +++ b/codex-rs/tui/src/conversation_history_widget.rs @@ -299,7 +299,6 @@ impl ConversationHistoryWidget { for entry in self.entries.iter_mut() { if let HistoryCell::ActiveMcpToolCall { call_id: history_id, - fq_tool_name, invocation, start, .. @@ -307,7 +306,7 @@ impl ConversationHistoryWidget { { if &call_id == history_id { let completed = HistoryCell::new_completed_mcp_tool_call( - fq_tool_name.clone(), + width, invocation.clone(), *start, success, diff --git a/codex-rs/tui/src/history_cell.rs b/codex-rs/tui/src/history_cell.rs index a1fc672c6b..481576b5b3 100644 --- a/codex-rs/tui/src/history_cell.rs +++ b/codex-rs/tui/src/history_cell.rs @@ -2,6 +2,7 @@ use crate::cell_widget::CellWidget; use crate::exec_command::escape_command; use crate::markdown::append_markdown; use crate::text_block::TextBlock; +use crate::text_formatting::format_and_truncate_tool_result; use base64::Engine; use codex_ansi_escape::ansi_escape_line; use codex_common::elapsed::format_duration; @@ -14,6 +15,7 @@ use image::DynamicImage; use image::GenericImageView; use image::ImageReader; use lazy_static::lazy_static; +use mcp_types::EmbeddedResourceResource; use ratatui::prelude::*; use ratatui::style::Color; use ratatui::style::Modifier; @@ -73,18 +75,14 @@ pub(crate) enum HistoryCell { /// An MCP tool call that has not finished yet. ActiveMcpToolCall { call_id: String, - /// `server.tool` fully-qualified name so we can show a concise label - fq_tool_name: String, - /// Formatted invocation that mirrors the `$ cmd ...` style of exec - /// commands. We keep this around so the completed state can reuse the - /// exact same text without re-formatting. - invocation: String, + /// Formatted line that shows the command name and arguments + invocation: Line<'static>, start: Instant, view: TextBlock, }, /// Completed MCP tool call where we show the result serialized as JSON. - CompletedMcpToolCallWithTextOutput { view: TextBlock }, + CompletedMcpToolCall { view: TextBlock }, /// Completed MCP tool call where the result is an image. /// Admittedly, [mcp_types::CallToolResult] can have multiple content types, @@ -289,8 +287,6 @@ impl HistoryCell { tool: String, arguments: Option, ) -> Self { - let fq_tool_name = format!("{server}.{tool}"); - // Format the arguments as compact JSON so they roughly fit on one // line. If there are no arguments we keep it empty so the invocation // mirrors a function-style call. @@ -302,29 +298,30 @@ impl HistoryCell { }) .unwrap_or_default(); - let invocation = if args_str.is_empty() { - format!("{fq_tool_name}()") - } else { - format!("{fq_tool_name}({args_str})") - }; + let invocation_spans = vec![ + Span::styled(server, Style::default().fg(Color::Blue)), + Span::raw("."), + Span::styled(tool, Style::default().fg(Color::Blue)), + Span::raw("("), + Span::styled(args_str, Style::default().fg(Color::Gray)), + Span::raw(")"), + ]; + let invocation = Line::from(invocation_spans); let start = Instant::now(); let title_line = Line::from(vec!["tool".magenta(), " running...".dim()]); - let lines: Vec> = vec![ - title_line, - Line::from(format!("$ {invocation}")), - Line::from(""), - ]; + let lines: Vec> = vec![title_line, invocation.clone(), Line::from("")]; HistoryCell::ActiveMcpToolCall { call_id, - fq_tool_name, invocation, start, view: TextBlock::new(lines), } } + /// If the first content is an image, return a new cell with the image. + /// TODO(rgwood-dd): Handle images properly even if they're not the first result. fn try_new_completed_mcp_tool_call_with_image_output( result: &Result, ) -> Option { @@ -370,8 +367,8 @@ impl HistoryCell { } pub(crate) fn new_completed_mcp_tool_call( - fq_tool_name: String, - invocation: String, + num_cols: u16, + invocation: Line<'static>, start: Instant, success: bool, result: Result, @@ -384,36 +381,70 @@ impl HistoryCell { let status_str = if success { "success" } else { "failed" }; let title_line = Line::from(vec![ "tool".magenta(), - format!(" {fq_tool_name} ({status_str}, duration: {})", duration).dim(), + " ".into(), + if success { + status_str.green() + } else { + status_str.red() + }, + format!(", duration: {duration}").gray(), ]); let mut lines: Vec> = Vec::new(); lines.push(title_line); - lines.push(Line::from(format!("$ {invocation}"))); - - // Convert result into serde_json::Value early so we don't have to - // worry about lifetimes inside the match arm. - let result_val = result.map(|r| { - serde_json::to_value(r) - .unwrap_or_else(|_| serde_json::Value::String("".into())) - }); - - if let Ok(res_val) = result_val { - let json_pretty = - serde_json::to_string_pretty(&res_val).unwrap_or_else(|_| res_val.to_string()); - let mut iter = json_pretty.lines(); - for raw in iter.by_ref().take(TOOL_CALL_MAX_LINES) { - lines.push(Line::from(raw.to_string()).dim()); + lines.push(invocation); + + match result { + Ok(mcp_types::CallToolResult { content, .. }) => { + if !content.is_empty() { + lines.push(Line::from("")); + + for tool_call_result in content { + let line_text = match tool_call_result { + mcp_types::CallToolResultContent::TextContent(text) => { + format_and_truncate_tool_result( + &text.text, + TOOL_CALL_MAX_LINES, + num_cols as usize, + ) + } + mcp_types::CallToolResultContent::ImageContent(_) => { + // TODO show images even if they're not the first result, will require a refactor of `CompletedMcpToolCall` + "".to_string() + } + mcp_types::CallToolResultContent::AudioContent(_) => { + "