mirror of
https://github.com/openai/codex.git
synced 2026-02-01 22:47:52 +00:00
Compare commits
29 Commits
subagents
...
wrap-for-s
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
930abf46e8 | ||
|
|
b242da8a41 | ||
|
|
6215f7fd54 | ||
|
|
17e5077507 | ||
|
|
b1079187e4 | ||
|
|
ae8f772ef2 | ||
|
|
468a8b4c38 | ||
|
|
cb32f9c64e | ||
|
|
907afc9425 | ||
|
|
7f7d1e30f3 | ||
|
|
568d6f819f | ||
|
|
251c4c2ba9 | ||
|
|
a6c346b9e1 | ||
|
|
e307040f10 | ||
|
|
7d67e54628 | ||
|
|
295ca27e98 | ||
|
|
7b20db942a | ||
|
|
ee2ccb5cb6 | ||
|
|
8b49346657 | ||
|
|
e49116a4c5 | ||
|
|
517ffd00c6 | ||
|
|
4157788310 | ||
|
|
32bbbbad61 | ||
|
|
c6a52d611c | ||
|
|
363636f5eb | ||
|
|
957d44918d | ||
|
|
eca97d8559 | ||
|
|
09819d9b47 | ||
|
|
e3b03eaccb |
91
codex-rs/Cargo.lock
generated
91
codex-rs/Cargo.lock
generated
@@ -635,6 +635,7 @@ name = "codex-apply-patch"
|
||||
version = "0.0.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"assert_cmd",
|
||||
"pretty_assertions",
|
||||
"similar",
|
||||
"tempfile",
|
||||
@@ -652,6 +653,7 @@ dependencies = [
|
||||
"codex-core",
|
||||
"codex-linux-sandbox",
|
||||
"dotenvy",
|
||||
"tempfile",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
@@ -731,6 +733,7 @@ dependencies = [
|
||||
"mime_guess",
|
||||
"openssl-sys",
|
||||
"os_info",
|
||||
"portable-pty",
|
||||
"predicates",
|
||||
"pretty_assertions",
|
||||
"rand 0.9.2",
|
||||
@@ -751,7 +754,7 @@ dependencies = [
|
||||
"tokio-test",
|
||||
"tokio-util",
|
||||
"toml 0.9.5",
|
||||
"toml_edit 0.23.3",
|
||||
"toml_edit 0.23.4",
|
||||
"tracing",
|
||||
"tree-sitter",
|
||||
"tree-sitter-bash",
|
||||
@@ -998,6 +1001,7 @@ dependencies = [
|
||||
"tui-markdown",
|
||||
"unicode-segmentation",
|
||||
"unicode-width 0.1.14",
|
||||
"url",
|
||||
"uuid",
|
||||
"vt100",
|
||||
]
|
||||
@@ -1479,6 +1483,12 @@ version = "0.15.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b"
|
||||
|
||||
[[package]]
|
||||
name = "downcast-rs"
|
||||
version = "1.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "75b325c5dbd37f80359721ad39aca5a29fb04c89279657cffdda8736d0c0b9d2"
|
||||
|
||||
[[package]]
|
||||
name = "dupe"
|
||||
version = "0.9.1"
|
||||
@@ -1724,6 +1734,17 @@ dependencies = [
|
||||
"simd-adler32",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "filedescriptor"
|
||||
version = "0.8.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e40758ed24c9b2eeb76c35fb0aebc66c626084edd827e07e1552279814c6682d"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"thiserror 1.0.69",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fixedbitset"
|
||||
version = "0.4.2"
|
||||
@@ -2702,6 +2723,7 @@ checksum = "4488594b9328dee448adb906d8b126d9b7deb7cf5c22161ee591610bb1be83c0"
|
||||
dependencies = [
|
||||
"bitflags 2.9.1",
|
||||
"libc",
|
||||
"redox_syscall",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3439,6 +3461,27 @@ dependencies = [
|
||||
"portable-atomic",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "portable-pty"
|
||||
version = "0.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b4a596a2b3d2752d94f51fac2d4a96737b8705dddd311a32b9af47211f08671e"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"bitflags 1.3.2",
|
||||
"downcast-rs",
|
||||
"filedescriptor",
|
||||
"lazy_static",
|
||||
"libc",
|
||||
"log",
|
||||
"nix",
|
||||
"serial2",
|
||||
"shared_library",
|
||||
"shell-words",
|
||||
"winapi",
|
||||
"winreg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "potential_utf"
|
||||
version = "0.1.2"
|
||||
@@ -4366,6 +4409,17 @@ dependencies = [
|
||||
"syn 2.0.104",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serial2"
|
||||
version = "0.2.31"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "26e1e5956803a69ddd72ce2de337b577898801528749565def03515f82bad5bb"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sha1"
|
||||
version = "0.10.6"
|
||||
@@ -4397,6 +4451,22 @@ dependencies = [
|
||||
"lazy_static",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "shared_library"
|
||||
version = "0.1.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5a9e7e0f2bfae24d8a5b5a66c5b257a83c7412304311512a0c054cd5e619da11"
|
||||
dependencies = [
|
||||
"lazy_static",
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "shell-words"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "24188a676b6ae68c3b2cb3a01be17fbf7240ce009799bb56d5b1409051e78fde"
|
||||
|
||||
[[package]]
|
||||
name = "shlex"
|
||||
version = "1.3.0"
|
||||
@@ -5126,9 +5196,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "toml_edit"
|
||||
version = "0.23.3"
|
||||
version = "0.23.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "17d3b47e6b7a040216ae5302712c94d1cf88c95b47efa80e2c59ce96c878267e"
|
||||
checksum = "7211ff1b8f0d3adae1663b7da9ffe396eabe1ca25f0b0bee42b0da29a9ddce93"
|
||||
dependencies = [
|
||||
"indexmap 2.10.0",
|
||||
"toml_datetime 0.7.0",
|
||||
@@ -5709,11 +5779,11 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "whoami"
|
||||
version = "1.6.0"
|
||||
version = "1.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6994d13118ab492c3c80c1f81928718159254c53c472bf9ce36f8dae4add02a7"
|
||||
checksum = "5d4a4db5077702ca3015d3d02d74974948aba2ad9e12ab7df718ee64ccd7e97d"
|
||||
dependencies = [
|
||||
"redox_syscall",
|
||||
"libredox",
|
||||
"wasite",
|
||||
"web-sys",
|
||||
]
|
||||
@@ -6176,6 +6246,15 @@ dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "winreg"
|
||||
version = "0.10.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d"
|
||||
dependencies = [
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "winsafe"
|
||||
version = "0.0.19"
|
||||
|
||||
@@ -43,6 +43,12 @@ To run Codex non-interactively, run `codex exec PROMPT` (you can also pass the p
|
||||
|
||||
Typing `@` triggers a fuzzy-filename search over the workspace root. Use up/down to select among the results and Tab or Enter to replace the `@` with the selected path. You can use Esc to cancel the search.
|
||||
|
||||
### Esc–Esc to edit a previous message
|
||||
|
||||
When the chat composer is empty, press Esc to prime “backtrack” mode. Press Esc again to open a transcript preview highlighting the last user message; press Esc repeatedly to step to older user messages. Press Enter to confirm and Codex will fork the conversation from that point, trim the visible transcript accordingly, and pre‑fill the composer with the selected user message so you can edit and resubmit it.
|
||||
|
||||
In the transcript preview, the footer shows an `Esc edit prev` hint while editing is active.
|
||||
|
||||
### `--cd`/`-C` flag
|
||||
|
||||
Sometimes it is not convenient to `cd` to the directory you want Codex to use as the "working root" before running Codex. Fortunately, `codex` supports a `--cd` option so you can specify whatever folder you want. You can confirm that Codex is honoring `--cd` by double-checking the **workdir** it reports in the TUI at the start of a new session.
|
||||
|
||||
@@ -7,6 +7,10 @@ version = { workspace = true }
|
||||
name = "codex_apply_patch"
|
||||
path = "src/lib.rs"
|
||||
|
||||
[[bin]]
|
||||
name = "apply_patch"
|
||||
path = "src/main.rs"
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
@@ -18,5 +22,6 @@ tree-sitter = "0.25.8"
|
||||
tree-sitter-bash = "0.25.0"
|
||||
|
||||
[dev-dependencies]
|
||||
assert_cmd = "2"
|
||||
pretty_assertions = "1.4.1"
|
||||
tempfile = "3.13.0"
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
mod parser;
|
||||
mod seek_sequence;
|
||||
mod standalone_executable;
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::path::Path;
|
||||
@@ -19,6 +20,8 @@ use tree_sitter::LanguageError;
|
||||
use tree_sitter::Parser;
|
||||
use tree_sitter_bash::LANGUAGE as BASH;
|
||||
|
||||
pub use standalone_executable::main;
|
||||
|
||||
/// Detailed instructions for gpt-4.1 on how to use the `apply_patch` tool.
|
||||
pub const APPLY_PATCH_TOOL_INSTRUCTIONS: &str = include_str!("../apply_patch_tool_instructions.md");
|
||||
|
||||
|
||||
3
codex-rs/apply-patch/src/main.rs
Normal file
3
codex-rs/apply-patch/src/main.rs
Normal file
@@ -0,0 +1,3 @@
|
||||
pub fn main() -> ! {
|
||||
codex_apply_patch::main()
|
||||
}
|
||||
59
codex-rs/apply-patch/src/standalone_executable.rs
Normal file
59
codex-rs/apply-patch/src/standalone_executable.rs
Normal file
@@ -0,0 +1,59 @@
|
||||
use std::io::Read;
|
||||
use std::io::Write;
|
||||
|
||||
pub fn main() -> ! {
|
||||
let exit_code = run_main();
|
||||
std::process::exit(exit_code);
|
||||
}
|
||||
|
||||
/// We would prefer to return `std::process::ExitCode`, but its `exit_process()`
|
||||
/// method is still a nightly API and we want main() to return !.
|
||||
pub fn run_main() -> i32 {
|
||||
// Expect either one argument (the full apply_patch payload) or read it from stdin.
|
||||
let mut args = std::env::args_os();
|
||||
let _argv0 = args.next();
|
||||
|
||||
let patch_arg = match args.next() {
|
||||
Some(arg) => match arg.into_string() {
|
||||
Ok(s) => s,
|
||||
Err(_) => {
|
||||
eprintln!("Error: apply_patch requires a UTF-8 PATCH argument.");
|
||||
return 1;
|
||||
}
|
||||
},
|
||||
None => {
|
||||
// No argument provided; attempt to read the patch from stdin.
|
||||
let mut buf = String::new();
|
||||
match std::io::stdin().read_to_string(&mut buf) {
|
||||
Ok(_) => {
|
||||
if buf.is_empty() {
|
||||
eprintln!("Usage: apply_patch 'PATCH'\n echo 'PATCH' | apply-patch");
|
||||
return 2;
|
||||
}
|
||||
buf
|
||||
}
|
||||
Err(err) => {
|
||||
eprintln!("Error: Failed to read PATCH from stdin.\n{err}");
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// Refuse extra args to avoid ambiguity.
|
||||
if args.next().is_some() {
|
||||
eprintln!("Error: apply_patch accepts exactly one argument.");
|
||||
return 2;
|
||||
}
|
||||
|
||||
let mut stdout = std::io::stdout();
|
||||
let mut stderr = std::io::stderr();
|
||||
match crate::apply_patch(&patch_arg, &mut stdout, &mut stderr) {
|
||||
Ok(()) => {
|
||||
// Flush to ensure output ordering when used in pipelines.
|
||||
let _ = stdout.flush();
|
||||
0
|
||||
}
|
||||
Err(_) => 1,
|
||||
}
|
||||
}
|
||||
3
codex-rs/apply-patch/tests/all.rs
Normal file
3
codex-rs/apply-patch/tests/all.rs
Normal file
@@ -0,0 +1,3 @@
|
||||
// Single integration test binary that aggregates all test modules.
|
||||
// The submodules live in `tests/suite/`.
|
||||
mod suite;
|
||||
90
codex-rs/apply-patch/tests/suite/cli.rs
Normal file
90
codex-rs/apply-patch/tests/suite/cli.rs
Normal file
@@ -0,0 +1,90 @@
|
||||
use assert_cmd::prelude::*;
|
||||
use std::fs;
|
||||
use std::process::Command;
|
||||
use tempfile::tempdir;
|
||||
|
||||
#[test]
|
||||
fn test_apply_patch_cli_add_and_update() -> anyhow::Result<()> {
|
||||
let tmp = tempdir()?;
|
||||
let file = "cli_test.txt";
|
||||
let absolute_path = tmp.path().join(file);
|
||||
|
||||
// 1) Add a file
|
||||
let add_patch = format!(
|
||||
r#"*** Begin Patch
|
||||
*** Add File: {file}
|
||||
+hello
|
||||
*** End Patch"#
|
||||
);
|
||||
Command::cargo_bin("apply_patch")
|
||||
.expect("should find apply_patch binary")
|
||||
.arg(add_patch)
|
||||
.current_dir(tmp.path())
|
||||
.assert()
|
||||
.success()
|
||||
.stdout(format!("Success. Updated the following files:\nA {file}\n"));
|
||||
assert_eq!(fs::read_to_string(&absolute_path)?, "hello\n");
|
||||
|
||||
// 2) Update the file
|
||||
let update_patch = format!(
|
||||
r#"*** Begin Patch
|
||||
*** Update File: {file}
|
||||
@@
|
||||
-hello
|
||||
+world
|
||||
*** End Patch"#
|
||||
);
|
||||
Command::cargo_bin("apply_patch")
|
||||
.expect("should find apply_patch binary")
|
||||
.arg(update_patch)
|
||||
.current_dir(tmp.path())
|
||||
.assert()
|
||||
.success()
|
||||
.stdout(format!("Success. Updated the following files:\nM {file}\n"));
|
||||
assert_eq!(fs::read_to_string(&absolute_path)?, "world\n");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_apply_patch_cli_stdin_add_and_update() -> anyhow::Result<()> {
|
||||
let tmp = tempdir()?;
|
||||
let file = "cli_test_stdin.txt";
|
||||
let absolute_path = tmp.path().join(file);
|
||||
|
||||
// 1) Add a file via stdin
|
||||
let add_patch = format!(
|
||||
r#"*** Begin Patch
|
||||
*** Add File: {file}
|
||||
+hello
|
||||
*** End Patch"#
|
||||
);
|
||||
let mut cmd =
|
||||
assert_cmd::Command::cargo_bin("apply_patch").expect("should find apply_patch binary");
|
||||
cmd.current_dir(tmp.path());
|
||||
cmd.write_stdin(add_patch)
|
||||
.assert()
|
||||
.success()
|
||||
.stdout(format!("Success. Updated the following files:\nA {file}\n"));
|
||||
assert_eq!(fs::read_to_string(&absolute_path)?, "hello\n");
|
||||
|
||||
// 2) Update the file via stdin
|
||||
let update_patch = format!(
|
||||
r#"*** Begin Patch
|
||||
*** Update File: {file}
|
||||
@@
|
||||
-hello
|
||||
+world
|
||||
*** End Patch"#
|
||||
);
|
||||
let mut cmd =
|
||||
assert_cmd::Command::cargo_bin("apply_patch").expect("should find apply_patch binary");
|
||||
cmd.current_dir(tmp.path());
|
||||
cmd.write_stdin(update_patch)
|
||||
.assert()
|
||||
.success()
|
||||
.stdout(format!("Success. Updated the following files:\nM {file}\n"));
|
||||
assert_eq!(fs::read_to_string(&absolute_path)?, "world\n");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
1
codex-rs/apply-patch/tests/suite/mod.rs
Normal file
1
codex-rs/apply-patch/tests/suite/mod.rs
Normal file
@@ -0,0 +1 @@
|
||||
mod cli;
|
||||
@@ -16,4 +16,5 @@ codex-apply-patch = { path = "../apply-patch" }
|
||||
codex-core = { path = "../core" }
|
||||
codex-linux-sandbox = { path = "../linux-sandbox" }
|
||||
dotenvy = "0.15.7"
|
||||
tempfile = "3"
|
||||
tokio = { version = "1", features = ["rt-multi-thread"] }
|
||||
|
||||
@@ -3,6 +3,13 @@ use std::path::Path;
|
||||
use std::path::PathBuf;
|
||||
|
||||
use codex_core::CODEX_APPLY_PATCH_ARG1;
|
||||
#[cfg(unix)]
|
||||
use std::os::unix::fs::symlink;
|
||||
use tempfile::TempDir;
|
||||
|
||||
const LINUX_SANDBOX_ARG0: &str = "codex-linux-sandbox";
|
||||
const APPLY_PATCH_ARG0: &str = "apply_patch";
|
||||
const MISSPELLED_APPLY_PATCH_ARG0: &str = "applypatch";
|
||||
|
||||
/// While we want to deploy the Codex CLI as a single executable for simplicity,
|
||||
/// we also want to expose some of its functionality as distinct CLIs, so we use
|
||||
@@ -39,9 +46,11 @@ where
|
||||
.and_then(|s| s.to_str())
|
||||
.unwrap_or("");
|
||||
|
||||
if exe_name == "codex-linux-sandbox" {
|
||||
if exe_name == LINUX_SANDBOX_ARG0 {
|
||||
// Safety: [`run_main`] never returns.
|
||||
codex_linux_sandbox::run_main();
|
||||
} else if exe_name == APPLY_PATCH_ARG0 || exe_name == MISSPELLED_APPLY_PATCH_ARG0 {
|
||||
codex_apply_patch::main();
|
||||
}
|
||||
|
||||
let argv1 = args.next().unwrap_or_default();
|
||||
@@ -68,6 +77,19 @@ where
|
||||
// before creating any threads/the Tokio runtime.
|
||||
load_dotenv();
|
||||
|
||||
// Retain the TempDir so it exists for the lifetime of the invocation of
|
||||
// this executable. Admittedly, we could invoke `keep()` on it, but it
|
||||
// would be nice to avoid leaving temporary directories behind, if possible.
|
||||
let _path_entry = match prepend_path_entry_for_apply_patch() {
|
||||
Ok(path_entry) => Some(path_entry),
|
||||
Err(err) => {
|
||||
// It is possible that Codex will proceed successfully even if
|
||||
// updating the PATH fails, so warn the user and move on.
|
||||
eprintln!("WARNING: proceeding, even though we could not update PATH: {err}");
|
||||
None
|
||||
}
|
||||
};
|
||||
|
||||
// Regular invocation – create a Tokio runtime and execute the provided
|
||||
// async entry-point.
|
||||
let runtime = tokio::runtime::Runtime::new()?;
|
||||
@@ -113,3 +135,67 @@ where
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Creates a temporary directory with either:
|
||||
///
|
||||
/// - UNIX: `apply_patch` symlink to the current executable
|
||||
/// - WINDOWS: `apply_patch.bat` batch script to invoke the current executable
|
||||
/// with the "secret" --codex-run-as-apply-patch flag.
|
||||
///
|
||||
/// This temporary directory is prepended to the PATH environment variable so
|
||||
/// that `apply_patch` can be on the PATH without requiring the user to
|
||||
/// install a separate `apply_patch` executable, simplifying the deployment of
|
||||
/// Codex CLI.
|
||||
///
|
||||
/// IMPORTANT: This function modifies the PATH environment variable, so it MUST
|
||||
/// be called before multiple threads are spawned.
|
||||
fn prepend_path_entry_for_apply_patch() -> std::io::Result<TempDir> {
|
||||
let temp_dir = TempDir::new()?;
|
||||
let path = temp_dir.path();
|
||||
|
||||
for filename in &[APPLY_PATCH_ARG0, MISSPELLED_APPLY_PATCH_ARG0] {
|
||||
let exe = std::env::current_exe()?;
|
||||
|
||||
#[cfg(unix)]
|
||||
{
|
||||
let link = path.join(filename);
|
||||
symlink(&exe, &link)?;
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
{
|
||||
let batch_script = path.join(format!("{filename}.bat"));
|
||||
std::fs::write(
|
||||
&batch_script,
|
||||
format!(
|
||||
r#"@echo off
|
||||
"{}" {CODEX_APPLY_PATCH_ARG1} %*
|
||||
"#,
|
||||
exe.display()
|
||||
),
|
||||
)?;
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
const PATH_SEPARATOR: &str = ":";
|
||||
|
||||
#[cfg(windows)]
|
||||
const PATH_SEPARATOR: &str = ";";
|
||||
|
||||
let path_element = path.display();
|
||||
let updated_path_env_var = match std::env::var("PATH") {
|
||||
Ok(existing_path) => {
|
||||
format!("{path_element}{PATH_SEPARATOR}{existing_path}")
|
||||
}
|
||||
Err(_) => {
|
||||
format!("{path_element}")
|
||||
}
|
||||
};
|
||||
|
||||
unsafe {
|
||||
std::env::set_var("PATH", updated_path_env_var);
|
||||
}
|
||||
|
||||
Ok(temp_dir)
|
||||
}
|
||||
|
||||
3
codex-rs/chatgpt/tests/all.rs
Normal file
3
codex-rs/chatgpt/tests/all.rs
Normal file
@@ -0,0 +1,3 @@
|
||||
// Single integration test binary that aggregates all test modules.
|
||||
// The submodules live in `tests/suite/`.
|
||||
mod suite;
|
||||
2
codex-rs/chatgpt/tests/suite/mod.rs
Normal file
2
codex-rs/chatgpt/tests/suite/mod.rs
Normal file
@@ -0,0 +1,2 @@
|
||||
// Aggregates all former standalone integration tests as modules.
|
||||
mod apply_command_e2e;
|
||||
@@ -6,6 +6,7 @@ version = { workspace = true }
|
||||
[lib]
|
||||
name = "codex_core"
|
||||
path = "src/lib.rs"
|
||||
doctest = false
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
@@ -28,6 +29,7 @@ libc = "0.2.175"
|
||||
mcp-types = { path = "../mcp-types" }
|
||||
mime_guess = "2.0"
|
||||
os_info = "3.12.0"
|
||||
portable-pty = "0.9.0"
|
||||
rand = "0.9"
|
||||
regex-lite = "0.1.6"
|
||||
reqwest = { version = "0.12", features = ["json", "stream"] }
|
||||
@@ -50,12 +52,12 @@ tokio = { version = "1", features = [
|
||||
] }
|
||||
tokio-util = "0.7.16"
|
||||
toml = "0.9.5"
|
||||
toml_edit = "0.23.3"
|
||||
toml_edit = "0.23.4"
|
||||
tracing = { version = "0.1.41", features = ["log"] }
|
||||
tree-sitter = "0.25.8"
|
||||
tree-sitter-bash = "0.25.0"
|
||||
uuid = { version = "1", features = ["serde", "v4"] }
|
||||
whoami = "1.6.0"
|
||||
whoami = "1.6.1"
|
||||
wildmatch = "2.4.0"
|
||||
|
||||
|
||||
|
||||
@@ -623,6 +623,12 @@ where
|
||||
Poll::Ready(Some(Ok(ResponseEvent::ReasoningSummaryPartAdded))) => {
|
||||
continue;
|
||||
}
|
||||
Poll::Ready(Some(Ok(ResponseEvent::WebSearchCallBegin { .. }))) => {
|
||||
return Poll::Ready(Some(Ok(ResponseEvent::WebSearchCallBegin {
|
||||
call_id: String::new(),
|
||||
query: None,
|
||||
})));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -149,7 +149,21 @@ impl ModelClient {
|
||||
let store = prompt.store && auth_mode != Some(AuthMode::ChatGPT);
|
||||
|
||||
let full_instructions = prompt.get_full_instructions(&self.config.model_family);
|
||||
let tools_json = create_tools_json_for_responses_api(&prompt.tools)?;
|
||||
let mut tools_json = create_tools_json_for_responses_api(&prompt.tools)?;
|
||||
// ChatGPT backend expects the preview name for web search.
|
||||
if auth_mode == Some(AuthMode::ChatGPT) {
|
||||
for tool in &mut tools_json {
|
||||
if let Some(map) = tool.as_object_mut()
|
||||
&& map.get("type").and_then(|v| v.as_str()) == Some("web_search")
|
||||
{
|
||||
map.insert(
|
||||
"type".to_string(),
|
||||
serde_json::Value::String("web_search_preview".to_string()),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let reasoning = create_reasoning_param_for_request(
|
||||
&self.config.model_family,
|
||||
self.effort,
|
||||
@@ -466,7 +480,8 @@ async fn process_sse<S>(
|
||||
}
|
||||
};
|
||||
|
||||
trace!("SSE event: {}", sse.data);
|
||||
let raw = sse.data.clone();
|
||||
trace!("SSE event: {}", raw);
|
||||
|
||||
let event: SseEvent = match serde_json::from_str(&sse.data) {
|
||||
Ok(event) => event,
|
||||
@@ -580,8 +595,24 @@ async fn process_sse<S>(
|
||||
| "response.in_progress"
|
||||
| "response.output_item.added"
|
||||
| "response.output_text.done" => {
|
||||
// Currently, we ignore this event, but we handle it
|
||||
// separately to skip the logging message in the `other` case.
|
||||
if event.kind == "response.output_item.added"
|
||||
&& let Some(item) = event.item.as_ref()
|
||||
{
|
||||
// Detect web_search_call begin and forward a synthetic event upstream.
|
||||
if let Some(ty) = item.get("type").and_then(|v| v.as_str())
|
||||
&& ty == "web_search_call"
|
||||
{
|
||||
let call_id = item
|
||||
.get("id")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
let ev = ResponseEvent::WebSearchCallBegin { call_id, query: None };
|
||||
if tx_event.send(Ok(ev)).await.is_err() {
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
"response.reasoning_summary_part.added" => {
|
||||
// Boundary between reasoning summary sections (e.g., titles).
|
||||
@@ -591,7 +622,7 @@ async fn process_sse<S>(
|
||||
}
|
||||
}
|
||||
"response.reasoning_summary_text.done" => {}
|
||||
other => debug!(other, "sse event"),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -93,6 +93,10 @@ pub enum ResponseEvent {
|
||||
ReasoningSummaryDelta(String),
|
||||
ReasoningContentDelta(String),
|
||||
ReasoningSummaryPartAdded,
|
||||
WebSearchCallBegin {
|
||||
call_id: String,
|
||||
query: Option<String>,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
|
||||
@@ -53,12 +53,18 @@ use crate::exec::SandboxType;
|
||||
use crate::exec::StdoutStream;
|
||||
use crate::exec::StreamOutput;
|
||||
use crate::exec::process_exec_tool_call;
|
||||
use crate::exec_command::EXEC_COMMAND_TOOL_NAME;
|
||||
use crate::exec_command::ExecCommandParams;
|
||||
use crate::exec_command::ExecSessionManager;
|
||||
use crate::exec_command::WRITE_STDIN_TOOL_NAME;
|
||||
use crate::exec_command::WriteStdinParams;
|
||||
use crate::exec_env::create_env;
|
||||
use crate::mcp_connection_manager::McpConnectionManager;
|
||||
use crate::mcp_tool_call::handle_mcp_tool_call;
|
||||
use crate::model_family::find_family_for_model;
|
||||
use crate::openai_tools::ApplyPatchToolArgs;
|
||||
use crate::openai_tools::ToolsConfig;
|
||||
use crate::openai_tools::ToolsConfigParams;
|
||||
use crate::openai_tools::get_openai_tools;
|
||||
use crate::parse_command::parse_command;
|
||||
use crate::plan_tool::handle_update_plan;
|
||||
@@ -91,6 +97,7 @@ use crate::protocol::StreamErrorEvent;
|
||||
use crate::protocol::Submission;
|
||||
use crate::protocol::TaskCompleteEvent;
|
||||
use crate::protocol::TurnDiffEvent;
|
||||
use crate::protocol::WebSearchBeginEvent;
|
||||
use crate::rollout::RolloutRecorder;
|
||||
use crate::safety::SafetyCheck;
|
||||
use crate::safety::assess_command_safety;
|
||||
@@ -142,6 +149,14 @@ pub struct CodexSpawnOk {
|
||||
}
|
||||
|
||||
pub(crate) const INITIAL_SUBMIT_ID: &str = "";
|
||||
pub(crate) const SUBMISSION_CHANNEL_CAPACITY: usize = 64;
|
||||
|
||||
// Model-formatting limits: clients get full streams; oonly content sent to the model is truncated.
|
||||
pub(crate) const MODEL_FORMAT_MAX_BYTES: usize = 10 * 1024; // 10 KiB
|
||||
pub(crate) const MODEL_FORMAT_MAX_LINES: usize = 256; // lines
|
||||
pub(crate) const MODEL_FORMAT_HEAD_LINES: usize = MODEL_FORMAT_MAX_LINES / 2;
|
||||
pub(crate) const MODEL_FORMAT_TAIL_LINES: usize = MODEL_FORMAT_MAX_LINES - MODEL_FORMAT_HEAD_LINES; // 128
|
||||
pub(crate) const MODEL_FORMAT_HEAD_BYTES: usize = MODEL_FORMAT_MAX_BYTES / 2;
|
||||
|
||||
impl Codex {
|
||||
/// Spawn a new [`Codex`] and initialize the session.
|
||||
@@ -150,7 +165,7 @@ impl Codex {
|
||||
auth_manager: Arc<AuthManager>,
|
||||
initial_history: Option<Vec<ResponseItem>>,
|
||||
) -> CodexResult<CodexSpawnOk> {
|
||||
let (tx_sub, rx_sub) = async_channel::bounded(64);
|
||||
let (tx_sub, rx_sub) = async_channel::bounded(SUBMISSION_CHANNEL_CAPACITY);
|
||||
let (tx_event, rx_event) = async_channel::unbounded();
|
||||
|
||||
let user_instructions = get_user_instructions(&config).await;
|
||||
@@ -254,6 +269,7 @@ pub(crate) struct Session {
|
||||
|
||||
/// Manager for external MCP servers/tools.
|
||||
mcp_connection_manager: McpConnectionManager,
|
||||
session_manager: ExecSessionManager,
|
||||
|
||||
/// External notifier command (will be passed as args to exec()). When
|
||||
/// `None` this feature is disabled.
|
||||
@@ -492,13 +508,15 @@ impl Session {
|
||||
);
|
||||
let turn_context = TurnContext {
|
||||
client,
|
||||
tools_config: ToolsConfig::new(
|
||||
&config.model_family,
|
||||
tools_config: ToolsConfig::new(&ToolsConfigParams {
|
||||
model_family: &config.model_family,
|
||||
approval_policy,
|
||||
sandbox_policy.clone(),
|
||||
config.include_plan_tool,
|
||||
config.include_apply_patch_tool,
|
||||
),
|
||||
sandbox_policy: sandbox_policy.clone(),
|
||||
include_plan_tool: config.include_plan_tool,
|
||||
include_apply_patch_tool: config.include_apply_patch_tool,
|
||||
include_web_search_request: config.tools_web_search_request,
|
||||
use_streamable_shell_tool: config.use_experimental_streamable_shell_tool,
|
||||
}),
|
||||
user_instructions,
|
||||
base_instructions,
|
||||
approval_policy,
|
||||
@@ -511,6 +529,7 @@ impl Session {
|
||||
session_id,
|
||||
tx_event: tx_event.clone(),
|
||||
mcp_connection_manager,
|
||||
session_manager: ExecSessionManager::default(),
|
||||
notify,
|
||||
state: Mutex::new(state),
|
||||
rollout: Mutex::new(rollout_recorder),
|
||||
@@ -722,15 +741,15 @@ impl Session {
|
||||
let ExecToolCallOutput {
|
||||
stdout,
|
||||
stderr,
|
||||
aggregated_output,
|
||||
duration,
|
||||
exit_code,
|
||||
} = output;
|
||||
// Because stdout and stderr could each be up to 100 KiB, we send
|
||||
// truncated versions.
|
||||
const MAX_STREAM_OUTPUT: usize = 5 * 1024; // 5KiB
|
||||
let stdout = stdout.text.chars().take(MAX_STREAM_OUTPUT).collect();
|
||||
let stderr = stderr.text.chars().take(MAX_STREAM_OUTPUT).collect();
|
||||
// Send full stdout/stderr to clients; do not truncate.
|
||||
let stdout = stdout.text.clone();
|
||||
let stderr = stderr.text.clone();
|
||||
let formatted_output = format_exec_output_str(output);
|
||||
let aggregated_output: String = aggregated_output.text.clone();
|
||||
|
||||
let msg = if is_apply_patch {
|
||||
EventMsg::PatchApplyEnd(PatchApplyEndEvent {
|
||||
@@ -744,9 +763,10 @@ impl Session {
|
||||
call_id: call_id.to_string(),
|
||||
stdout,
|
||||
stderr,
|
||||
formatted_output,
|
||||
duration: *duration,
|
||||
aggregated_output,
|
||||
exit_code: *exit_code,
|
||||
duration: *duration,
|
||||
formatted_output,
|
||||
})
|
||||
};
|
||||
|
||||
@@ -804,6 +824,7 @@ impl Session {
|
||||
exit_code: -1,
|
||||
stdout: StreamOutput::new(String::new()),
|
||||
stderr: StreamOutput::new(get_error_message_ui(e)),
|
||||
aggregated_output: StreamOutput::new(get_error_message_ui(e)),
|
||||
duration: Duration::default(),
|
||||
};
|
||||
&output_stderr
|
||||
@@ -1074,13 +1095,15 @@ async fn submission_loop(
|
||||
.unwrap_or(prev.sandbox_policy.clone());
|
||||
let new_cwd = cwd.clone().unwrap_or_else(|| prev.cwd.clone());
|
||||
|
||||
let tools_config = ToolsConfig::new(
|
||||
&effective_family,
|
||||
new_approval_policy,
|
||||
new_sandbox_policy.clone(),
|
||||
config.include_plan_tool,
|
||||
config.include_apply_patch_tool,
|
||||
);
|
||||
let tools_config = ToolsConfig::new(&ToolsConfigParams {
|
||||
model_family: &effective_family,
|
||||
approval_policy: new_approval_policy,
|
||||
sandbox_policy: new_sandbox_policy.clone(),
|
||||
include_plan_tool: config.include_plan_tool,
|
||||
include_apply_patch_tool: config.include_apply_patch_tool,
|
||||
include_web_search_request: config.tools_web_search_request,
|
||||
use_streamable_shell_tool: config.use_experimental_streamable_shell_tool,
|
||||
});
|
||||
|
||||
let new_turn_context = TurnContext {
|
||||
client,
|
||||
@@ -1129,6 +1152,7 @@ async fn submission_loop(
|
||||
if let Err(items) = sess.inject_input(items) {
|
||||
// Derive a fresh TurnContext for this turn using the provided overrides.
|
||||
let provider = turn_context.client.get_provider();
|
||||
let auth_manager = turn_context.client.get_auth_manager();
|
||||
|
||||
// Derive a model family for the requested model; fall back to the session's.
|
||||
let model_family = find_family_for_model(&model)
|
||||
@@ -1143,7 +1167,7 @@ async fn submission_loop(
|
||||
// Reuse the same provider and session id; auth defaults to env/API key.
|
||||
let client = ModelClient::new(
|
||||
Arc::new(per_turn_config),
|
||||
None,
|
||||
auth_manager,
|
||||
provider,
|
||||
effort,
|
||||
summary,
|
||||
@@ -1152,13 +1176,16 @@ async fn submission_loop(
|
||||
|
||||
let fresh_turn_context = TurnContext {
|
||||
client,
|
||||
tools_config: ToolsConfig::new(
|
||||
&model_family,
|
||||
tools_config: ToolsConfig::new(&ToolsConfigParams {
|
||||
model_family: &model_family,
|
||||
approval_policy,
|
||||
sandbox_policy.clone(),
|
||||
config.include_plan_tool,
|
||||
config.include_apply_patch_tool,
|
||||
),
|
||||
sandbox_policy: sandbox_policy.clone(),
|
||||
include_plan_tool: config.include_plan_tool,
|
||||
include_apply_patch_tool: config.include_apply_patch_tool,
|
||||
include_web_search_request: config.tools_web_search_request,
|
||||
use_streamable_shell_tool: config
|
||||
.use_experimental_streamable_shell_tool,
|
||||
}),
|
||||
user_instructions: turn_context.user_instructions.clone(),
|
||||
base_instructions: turn_context.base_instructions.clone(),
|
||||
approval_policy,
|
||||
@@ -1669,6 +1696,7 @@ async fn try_run_turn(
|
||||
let mut stream = turn_context.client.clone().stream(&prompt).await?;
|
||||
|
||||
let mut output = Vec::new();
|
||||
|
||||
loop {
|
||||
// Poll the next item from the model stream. We must inspect *both* Ok and Err
|
||||
// cases so that transient stream failures (e.g., dropped SSE connection before
|
||||
@@ -1705,6 +1733,16 @@ async fn try_run_turn(
|
||||
.await?;
|
||||
output.push(ProcessedResponseItem { item, response });
|
||||
}
|
||||
ResponseEvent::WebSearchCallBegin { call_id, query } => {
|
||||
let q = query.unwrap_or_else(|| "Searching Web...".to_string());
|
||||
let _ = sess
|
||||
.tx_event
|
||||
.send(Event {
|
||||
id: sub_id.to_string(),
|
||||
msg: EventMsg::WebSearchBegin(WebSearchBeginEvent { call_id, query: q }),
|
||||
})
|
||||
.await;
|
||||
}
|
||||
ResponseEvent::Completed {
|
||||
response_id: _,
|
||||
token_usage,
|
||||
@@ -2063,6 +2101,54 @@ async fn handle_function_call(
|
||||
.await
|
||||
}
|
||||
"update_plan" => handle_update_plan(sess, arguments, sub_id, call_id).await,
|
||||
EXEC_COMMAND_TOOL_NAME => {
|
||||
// TODO(mbolin): Sandbox check.
|
||||
let exec_params = match serde_json::from_str::<ExecCommandParams>(&arguments) {
|
||||
Ok(params) => params,
|
||||
Err(e) => {
|
||||
return ResponseInputItem::FunctionCallOutput {
|
||||
call_id,
|
||||
output: FunctionCallOutputPayload {
|
||||
content: format!("failed to parse function arguments: {e}"),
|
||||
success: Some(false),
|
||||
},
|
||||
};
|
||||
}
|
||||
};
|
||||
let result = sess
|
||||
.session_manager
|
||||
.handle_exec_command_request(exec_params)
|
||||
.await;
|
||||
let function_call_output = crate::exec_command::result_into_payload(result);
|
||||
ResponseInputItem::FunctionCallOutput {
|
||||
call_id,
|
||||
output: function_call_output,
|
||||
}
|
||||
}
|
||||
WRITE_STDIN_TOOL_NAME => {
|
||||
let write_stdin_params = match serde_json::from_str::<WriteStdinParams>(&arguments) {
|
||||
Ok(params) => params,
|
||||
Err(e) => {
|
||||
return ResponseInputItem::FunctionCallOutput {
|
||||
call_id,
|
||||
output: FunctionCallOutputPayload {
|
||||
content: format!("failed to parse function arguments: {e}"),
|
||||
success: Some(false),
|
||||
},
|
||||
};
|
||||
}
|
||||
};
|
||||
let result = sess
|
||||
.session_manager
|
||||
.handle_write_stdin_request(write_stdin_params)
|
||||
.await;
|
||||
let function_call_output: FunctionCallOutputPayload =
|
||||
crate::exec_command::result_into_payload(result);
|
||||
ResponseInputItem::FunctionCallOutput {
|
||||
call_id,
|
||||
output: function_call_output,
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
match sess.mcp_connection_manager.parse_tool_name(&name) {
|
||||
Some((server, tool_name)) => {
|
||||
@@ -2550,23 +2636,103 @@ async fn handle_sandbox_error(
|
||||
|
||||
fn format_exec_output_str(exec_output: &ExecToolCallOutput) -> String {
|
||||
let ExecToolCallOutput {
|
||||
exit_code,
|
||||
stdout,
|
||||
stderr,
|
||||
..
|
||||
aggregated_output, ..
|
||||
} = exec_output;
|
||||
|
||||
let is_success = *exit_code == 0;
|
||||
let output = if is_success { stdout } else { stderr };
|
||||
// Head+tail truncation for the model: show the beginning and end with an elision.
|
||||
// Clients still receive full streams; only this formatted summary is capped.
|
||||
|
||||
let mut formatted_output = output.text.clone();
|
||||
if let Some(truncated_after_lines) = output.truncated_after_lines {
|
||||
formatted_output.push_str(&format!(
|
||||
"\n\n[Output truncated after {truncated_after_lines} lines: too many lines or bytes.]",
|
||||
));
|
||||
let s = aggregated_output.text.as_str();
|
||||
let total_lines = s.lines().count();
|
||||
if s.len() <= MODEL_FORMAT_MAX_BYTES && total_lines <= MODEL_FORMAT_MAX_LINES {
|
||||
return s.to_string();
|
||||
}
|
||||
|
||||
formatted_output
|
||||
let lines: Vec<&str> = s.lines().collect();
|
||||
let head_take = MODEL_FORMAT_HEAD_LINES.min(lines.len());
|
||||
let tail_take = MODEL_FORMAT_TAIL_LINES.min(lines.len().saturating_sub(head_take));
|
||||
let omitted = lines.len().saturating_sub(head_take + tail_take);
|
||||
|
||||
// Join head and tail blocks (lines() strips newlines; reinsert them)
|
||||
let head_block = lines
|
||||
.iter()
|
||||
.take(head_take)
|
||||
.cloned()
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n");
|
||||
let tail_block = if tail_take > 0 {
|
||||
lines[lines.len() - tail_take..].join("\n")
|
||||
} else {
|
||||
String::new()
|
||||
};
|
||||
let marker = format!("\n[... omitted {omitted} of {total_lines} lines ...]\n\n");
|
||||
|
||||
// Byte budgets for head/tail around the marker
|
||||
let mut head_budget = MODEL_FORMAT_HEAD_BYTES.min(MODEL_FORMAT_MAX_BYTES);
|
||||
let tail_budget = MODEL_FORMAT_MAX_BYTES.saturating_sub(head_budget + marker.len());
|
||||
if tail_budget == 0 && marker.len() >= MODEL_FORMAT_MAX_BYTES {
|
||||
// Degenerate case: marker alone exceeds budget; return a clipped marker
|
||||
return take_bytes_at_char_boundary(&marker, MODEL_FORMAT_MAX_BYTES).to_string();
|
||||
}
|
||||
if tail_budget == 0 {
|
||||
// Make room for the marker by shrinking head
|
||||
head_budget = MODEL_FORMAT_MAX_BYTES.saturating_sub(marker.len());
|
||||
}
|
||||
|
||||
// Enforce line-count cap by trimming head/tail lines
|
||||
let head_lines_text = head_block;
|
||||
let tail_lines_text = tail_block;
|
||||
// Build final string respecting byte budgets
|
||||
let head_part = take_bytes_at_char_boundary(&head_lines_text, head_budget);
|
||||
let mut result = String::with_capacity(MODEL_FORMAT_MAX_BYTES.min(s.len()));
|
||||
result.push_str(head_part);
|
||||
result.push_str(&marker);
|
||||
|
||||
let remaining = MODEL_FORMAT_MAX_BYTES.saturating_sub(result.len());
|
||||
let tail_budget_final = remaining;
|
||||
let tail_part = take_last_bytes_at_char_boundary(&tail_lines_text, tail_budget_final);
|
||||
result.push_str(tail_part);
|
||||
|
||||
result
|
||||
}
|
||||
|
||||
// Truncate a &str to a byte budget at a char boundary (prefix)
|
||||
#[inline]
|
||||
fn take_bytes_at_char_boundary(s: &str, maxb: usize) -> &str {
|
||||
if s.len() <= maxb {
|
||||
return s;
|
||||
}
|
||||
let mut last_ok = 0;
|
||||
for (i, ch) in s.char_indices() {
|
||||
let nb = i + ch.len_utf8();
|
||||
if nb > maxb {
|
||||
break;
|
||||
}
|
||||
last_ok = nb;
|
||||
}
|
||||
&s[..last_ok]
|
||||
}
|
||||
|
||||
// Take a suffix of a &str within a byte budget at a char boundary
|
||||
#[inline]
|
||||
fn take_last_bytes_at_char_boundary(s: &str, maxb: usize) -> &str {
|
||||
if s.len() <= maxb {
|
||||
return s;
|
||||
}
|
||||
let mut start = s.len();
|
||||
let mut used = 0usize;
|
||||
for (i, ch) in s.char_indices().rev() {
|
||||
let nb = ch.len_utf8();
|
||||
if used + nb > maxb {
|
||||
break;
|
||||
}
|
||||
start = i;
|
||||
used += nb;
|
||||
if start == 0 {
|
||||
break;
|
||||
}
|
||||
}
|
||||
&s[start..]
|
||||
}
|
||||
|
||||
/// Exec output is a pre-serialized JSON payload
|
||||
@@ -2651,15 +2817,9 @@ async fn drain_to_completed(
|
||||
response_id: _,
|
||||
token_usage,
|
||||
}) => {
|
||||
let token_usage = match token_usage {
|
||||
Some(usage) => usage,
|
||||
None => {
|
||||
return Err(CodexErr::Stream(
|
||||
"token_usage was None in ResponseEvent::Completed".into(),
|
||||
None,
|
||||
));
|
||||
}
|
||||
};
|
||||
// some providers don't return token usage, so we default
|
||||
// TODO: consider approximate token usage
|
||||
let token_usage = token_usage.unwrap_or_default();
|
||||
sess.tx_event
|
||||
.send(Event {
|
||||
id: sub_id.to_string(),
|
||||
@@ -2667,6 +2827,7 @@ async fn drain_to_completed(
|
||||
})
|
||||
.await
|
||||
.ok();
|
||||
|
||||
return Ok(());
|
||||
}
|
||||
Ok(_) => continue,
|
||||
@@ -2717,6 +2878,7 @@ mod tests {
|
||||
use mcp_types::TextContent;
|
||||
use pretty_assertions::assert_eq;
|
||||
use serde_json::json;
|
||||
use std::time::Duration as StdDuration;
|
||||
|
||||
fn text_block(s: &str) -> ContentBlock {
|
||||
ContentBlock::TextContent(TextContent {
|
||||
@@ -2751,6 +2913,82 @@ mod tests {
|
||||
assert_eq!(expected, got);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn model_truncation_head_tail_by_lines() {
|
||||
// Build 400 short lines so line-count limit, not byte budget, triggers truncation
|
||||
let lines: Vec<String> = (1..=400).map(|i| format!("line{i}")).collect();
|
||||
let full = lines.join("\n");
|
||||
|
||||
let exec = ExecToolCallOutput {
|
||||
exit_code: 0,
|
||||
stdout: StreamOutput::new(String::new()),
|
||||
stderr: StreamOutput::new(String::new()),
|
||||
aggregated_output: StreamOutput::new(full.clone()),
|
||||
duration: StdDuration::from_secs(1),
|
||||
};
|
||||
|
||||
let out = format_exec_output_str(&exec);
|
||||
|
||||
// Expect elision marker with correct counts
|
||||
let omitted = 400 - MODEL_FORMAT_MAX_LINES; // 144
|
||||
let marker = format!("\n[... omitted {omitted} of 400 lines ...]\n\n");
|
||||
assert!(out.contains(&marker), "missing marker: {out}");
|
||||
|
||||
// Validate head and tail
|
||||
let parts: Vec<&str> = out.split(&marker).collect();
|
||||
assert_eq!(parts.len(), 2, "expected one marker split");
|
||||
let head = parts[0];
|
||||
let tail = parts[1];
|
||||
|
||||
let expected_head: String = (1..=MODEL_FORMAT_HEAD_LINES)
|
||||
.map(|i| format!("line{i}"))
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n");
|
||||
assert!(head.starts_with(&expected_head), "head mismatch");
|
||||
|
||||
let expected_tail: String = ((400 - MODEL_FORMAT_TAIL_LINES + 1)..=400)
|
||||
.map(|i| format!("line{i}"))
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n");
|
||||
assert!(tail.ends_with(&expected_tail), "tail mismatch");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn model_truncation_respects_byte_budget() {
|
||||
// Construct a large output (about 100kB) so byte budget dominates
|
||||
let big_line = "x".repeat(100);
|
||||
let full = std::iter::repeat_n(big_line.clone(), 1000)
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n");
|
||||
|
||||
let exec = ExecToolCallOutput {
|
||||
exit_code: 0,
|
||||
stdout: StreamOutput::new(String::new()),
|
||||
stderr: StreamOutput::new(String::new()),
|
||||
aggregated_output: StreamOutput::new(full.clone()),
|
||||
duration: StdDuration::from_secs(1),
|
||||
};
|
||||
|
||||
let out = format_exec_output_str(&exec);
|
||||
assert!(out.len() <= MODEL_FORMAT_MAX_BYTES, "exceeds byte budget");
|
||||
assert!(out.contains("omitted"), "should contain elision marker");
|
||||
|
||||
// Ensure head and tail are drawn from the original
|
||||
assert!(full.starts_with(out.chars().take(8).collect::<String>().as_str()));
|
||||
assert!(
|
||||
full.ends_with(
|
||||
out.chars()
|
||||
.rev()
|
||||
.take(8)
|
||||
.collect::<String>()
|
||||
.chars()
|
||||
.rev()
|
||||
.collect::<String>()
|
||||
.as_str()
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn falls_back_to_content_when_structured_is_null() {
|
||||
let ctr = CallToolResult {
|
||||
|
||||
@@ -169,11 +169,15 @@ pub struct Config {
|
||||
/// model family's default preference.
|
||||
pub include_apply_patch_tool: bool,
|
||||
|
||||
pub tools_web_search_request: bool,
|
||||
|
||||
/// The value for the `originator` header included with Responses API requests.
|
||||
pub responses_originator_header: String,
|
||||
|
||||
/// If set to `true`, the API key will be signed with the `originator` header.
|
||||
pub preferred_auth_method: AuthMode,
|
||||
|
||||
pub use_experimental_streamable_shell_tool: bool,
|
||||
}
|
||||
|
||||
impl Config {
|
||||
@@ -469,6 +473,8 @@ pub struct ConfigToml {
|
||||
/// Experimental path to a file whose contents replace the built-in BASE_INSTRUCTIONS.
|
||||
pub experimental_instructions_file: Option<PathBuf>,
|
||||
|
||||
pub experimental_use_exec_command_tool: Option<bool>,
|
||||
|
||||
/// The value for the `originator` header included with Responses API requests.
|
||||
pub responses_originator_header_internal_override: Option<String>,
|
||||
|
||||
@@ -476,6 +482,9 @@ pub struct ConfigToml {
|
||||
|
||||
/// If set to `true`, the API key will be signed with the `originator` header.
|
||||
pub preferred_auth_method: Option<AuthMode>,
|
||||
|
||||
/// Nested tools section for feature toggles
|
||||
pub tools: Option<ToolsToml>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, Clone, PartialEq, Eq)]
|
||||
@@ -483,6 +492,13 @@ pub struct ProjectConfig {
|
||||
pub trust_level: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, Clone, Default)]
|
||||
pub struct ToolsToml {
|
||||
// Renamed from `web_search_request`; keep alias for backwards compatibility.
|
||||
#[serde(default, alias = "web_search_request")]
|
||||
pub web_search: Option<bool>,
|
||||
}
|
||||
|
||||
impl ConfigToml {
|
||||
/// Derive the effective sandbox policy from the configuration.
|
||||
fn derive_sandbox_policy(&self, sandbox_mode_override: Option<SandboxMode>) -> SandboxPolicy {
|
||||
@@ -572,6 +588,7 @@ pub struct ConfigOverrides {
|
||||
pub include_apply_patch_tool: Option<bool>,
|
||||
pub disable_response_storage: Option<bool>,
|
||||
pub show_raw_agent_reasoning: Option<bool>,
|
||||
pub tools_web_search_request: Option<bool>,
|
||||
}
|
||||
|
||||
impl Config {
|
||||
@@ -598,6 +615,7 @@ impl Config {
|
||||
include_apply_patch_tool,
|
||||
disable_response_storage,
|
||||
show_raw_agent_reasoning,
|
||||
tools_web_search_request: override_tools_web_search_request,
|
||||
} = overrides;
|
||||
|
||||
let config_profile = match config_profile_key.as_ref().or(cfg.profile.as_ref()) {
|
||||
@@ -636,7 +654,7 @@ impl Config {
|
||||
})?
|
||||
.clone();
|
||||
|
||||
let shell_environment_policy = cfg.shell_environment_policy.into();
|
||||
let shell_environment_policy = cfg.shell_environment_policy.clone().into();
|
||||
|
||||
let resolved_cwd = {
|
||||
use std::env;
|
||||
@@ -657,7 +675,11 @@ impl Config {
|
||||
}
|
||||
};
|
||||
|
||||
let history = cfg.history.unwrap_or_default();
|
||||
let history = cfg.history.clone().unwrap_or_default();
|
||||
|
||||
let tools_web_search_request = override_tools_web_search_request
|
||||
.or(cfg.tools.as_ref().and_then(|t| t.web_search))
|
||||
.unwrap_or(false);
|
||||
|
||||
let model = model
|
||||
.or(config_profile.model)
|
||||
@@ -731,7 +753,7 @@ impl Config {
|
||||
codex_home,
|
||||
history,
|
||||
file_opener: cfg.file_opener.unwrap_or(UriBasedFileOpener::VsCode),
|
||||
tui: cfg.tui.unwrap_or_default(),
|
||||
tui: cfg.tui.clone().unwrap_or_default(),
|
||||
codex_linux_sandbox_exe,
|
||||
|
||||
hide_agent_reasoning: cfg.hide_agent_reasoning.unwrap_or(false),
|
||||
@@ -750,14 +772,18 @@ impl Config {
|
||||
model_verbosity: config_profile.model_verbosity.or(cfg.model_verbosity),
|
||||
chatgpt_base_url: config_profile
|
||||
.chatgpt_base_url
|
||||
.or(cfg.chatgpt_base_url)
|
||||
.or(cfg.chatgpt_base_url.clone())
|
||||
.unwrap_or("https://chatgpt.com/backend-api/".to_string()),
|
||||
|
||||
experimental_resume,
|
||||
include_plan_tool: include_plan_tool.unwrap_or(false),
|
||||
include_apply_patch_tool: include_apply_patch_tool.unwrap_or(false),
|
||||
tools_web_search_request,
|
||||
responses_originator_header,
|
||||
preferred_auth_method: cfg.preferred_auth_method.unwrap_or(AuthMode::ChatGPT),
|
||||
use_experimental_streamable_shell_tool: cfg
|
||||
.experimental_use_exec_command_tool
|
||||
.unwrap_or(false),
|
||||
};
|
||||
Ok(config)
|
||||
}
|
||||
@@ -1122,8 +1148,10 @@ disable_response_storage = true
|
||||
base_instructions: None,
|
||||
include_plan_tool: false,
|
||||
include_apply_patch_tool: false,
|
||||
tools_web_search_request: false,
|
||||
responses_originator_header: "codex_cli_rs".to_string(),
|
||||
preferred_auth_method: AuthMode::ChatGPT,
|
||||
use_experimental_streamable_shell_tool: false,
|
||||
},
|
||||
o3_profile_config
|
||||
);
|
||||
@@ -1176,8 +1204,10 @@ disable_response_storage = true
|
||||
base_instructions: None,
|
||||
include_plan_tool: false,
|
||||
include_apply_patch_tool: false,
|
||||
tools_web_search_request: false,
|
||||
responses_originator_header: "codex_cli_rs".to_string(),
|
||||
preferred_auth_method: AuthMode::ChatGPT,
|
||||
use_experimental_streamable_shell_tool: false,
|
||||
};
|
||||
|
||||
assert_eq!(expected_gpt3_profile_config, gpt3_profile_config);
|
||||
@@ -1245,8 +1275,10 @@ disable_response_storage = true
|
||||
base_instructions: None,
|
||||
include_plan_tool: false,
|
||||
include_apply_patch_tool: false,
|
||||
tools_web_search_request: false,
|
||||
responses_originator_header: "codex_cli_rs".to_string(),
|
||||
preferred_auth_method: AuthMode::ChatGPT,
|
||||
use_experimental_streamable_shell_tool: false,
|
||||
};
|
||||
|
||||
assert_eq!(expected_zdr_profile_config, zdr_profile_config);
|
||||
|
||||
@@ -181,6 +181,8 @@ impl CodexErr {
|
||||
pub fn get_error_message_ui(e: &CodexErr) -> String {
|
||||
match e {
|
||||
CodexErr::Sandbox(SandboxErr::Denied(_, _, stderr)) => stderr.to_string(),
|
||||
// Timeouts are not sandbox errors from a UX perspective; present them plainly
|
||||
CodexErr::Sandbox(SandboxErr::Timeout) => "error: command timed out".to_string(),
|
||||
_ => e.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -28,18 +28,17 @@ use crate::spawn::StdioPolicy;
|
||||
use crate::spawn::spawn_child_async;
|
||||
use serde_bytes::ByteBuf;
|
||||
|
||||
// Maximum we send for each stream, which is either:
|
||||
// - 10KiB OR
|
||||
// - 256 lines
|
||||
const MAX_STREAM_OUTPUT: usize = 10 * 1024;
|
||||
const MAX_STREAM_OUTPUT_LINES: usize = 256;
|
||||
|
||||
const DEFAULT_TIMEOUT_MS: u64 = 10_000;
|
||||
|
||||
// Hardcode these since it does not seem worth including the libc crate just
|
||||
// for these.
|
||||
const SIGKILL_CODE: i32 = 9;
|
||||
const TIMEOUT_CODE: i32 = 64;
|
||||
const EXIT_CODE_SIGNAL_BASE: i32 = 128; // conventional shell: 128 + signal
|
||||
|
||||
// I/O buffer sizing
|
||||
const READ_CHUNK_SIZE: usize = 8192; // bytes per read
|
||||
const AGGREGATE_BUFFER_INITIAL_CAPACITY: usize = 8 * 1024; // 8 KiB
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ExecParams {
|
||||
@@ -153,6 +152,7 @@ pub async fn process_exec_tool_call(
|
||||
exit_code,
|
||||
stdout,
|
||||
stderr,
|
||||
aggregated_output: raw_output.aggregated_output.from_utf8_lossy(),
|
||||
duration,
|
||||
})
|
||||
}
|
||||
@@ -189,10 +189,11 @@ pub struct StreamOutput<T> {
|
||||
pub truncated_after_lines: Option<u32>,
|
||||
}
|
||||
#[derive(Debug)]
|
||||
pub struct RawExecToolCallOutput {
|
||||
struct RawExecToolCallOutput {
|
||||
pub exit_status: ExitStatus,
|
||||
pub stdout: StreamOutput<Vec<u8>>,
|
||||
pub stderr: StreamOutput<Vec<u8>>,
|
||||
pub aggregated_output: StreamOutput<Vec<u8>>,
|
||||
}
|
||||
|
||||
impl StreamOutput<String> {
|
||||
@@ -213,11 +214,17 @@ impl StreamOutput<Vec<u8>> {
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn append_all(dst: &mut Vec<u8>, src: &[u8]) {
|
||||
dst.extend_from_slice(src);
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct ExecToolCallOutput {
|
||||
pub exit_code: i32,
|
||||
pub stdout: StreamOutput<String>,
|
||||
pub stderr: StreamOutput<String>,
|
||||
pub aggregated_output: StreamOutput<String>,
|
||||
pub duration: Duration,
|
||||
}
|
||||
|
||||
@@ -253,7 +260,7 @@ async fn exec(
|
||||
|
||||
/// Consumes the output of a child process, truncating it so it is suitable for
|
||||
/// use as the output of a `shell` tool call. Also enforces specified timeout.
|
||||
pub(crate) async fn consume_truncated_output(
|
||||
async fn consume_truncated_output(
|
||||
mut child: Child,
|
||||
timeout: Duration,
|
||||
stdout_stream: Option<StdoutStream>,
|
||||
@@ -273,19 +280,19 @@ pub(crate) async fn consume_truncated_output(
|
||||
))
|
||||
})?;
|
||||
|
||||
let (agg_tx, agg_rx) = async_channel::unbounded::<Vec<u8>>();
|
||||
|
||||
let stdout_handle = tokio::spawn(read_capped(
|
||||
BufReader::new(stdout_reader),
|
||||
MAX_STREAM_OUTPUT,
|
||||
MAX_STREAM_OUTPUT_LINES,
|
||||
stdout_stream.clone(),
|
||||
false,
|
||||
Some(agg_tx.clone()),
|
||||
));
|
||||
let stderr_handle = tokio::spawn(read_capped(
|
||||
BufReader::new(stderr_reader),
|
||||
MAX_STREAM_OUTPUT,
|
||||
MAX_STREAM_OUTPUT_LINES,
|
||||
stdout_stream.clone(),
|
||||
true,
|
||||
Some(agg_tx.clone()),
|
||||
));
|
||||
|
||||
let exit_status = tokio::select! {
|
||||
@@ -297,38 +304,48 @@ pub(crate) async fn consume_truncated_output(
|
||||
// timeout
|
||||
child.start_kill()?;
|
||||
// Debatable whether `child.wait().await` should be called here.
|
||||
synthetic_exit_status(128 + TIMEOUT_CODE)
|
||||
synthetic_exit_status(EXIT_CODE_SIGNAL_BASE + TIMEOUT_CODE)
|
||||
}
|
||||
}
|
||||
}
|
||||
_ = tokio::signal::ctrl_c() => {
|
||||
child.start_kill()?;
|
||||
synthetic_exit_status(128 + SIGKILL_CODE)
|
||||
synthetic_exit_status(EXIT_CODE_SIGNAL_BASE + SIGKILL_CODE)
|
||||
}
|
||||
};
|
||||
|
||||
let stdout = stdout_handle.await??;
|
||||
let stderr = stderr_handle.await??;
|
||||
|
||||
drop(agg_tx);
|
||||
|
||||
let mut combined_buf = Vec::with_capacity(AGGREGATE_BUFFER_INITIAL_CAPACITY);
|
||||
while let Ok(chunk) = agg_rx.recv().await {
|
||||
append_all(&mut combined_buf, &chunk);
|
||||
}
|
||||
let aggregated_output = StreamOutput {
|
||||
text: combined_buf,
|
||||
truncated_after_lines: None,
|
||||
};
|
||||
|
||||
Ok(RawExecToolCallOutput {
|
||||
exit_status,
|
||||
stdout,
|
||||
stderr,
|
||||
aggregated_output,
|
||||
})
|
||||
}
|
||||
|
||||
async fn read_capped<R: AsyncRead + Unpin + Send + 'static>(
|
||||
mut reader: R,
|
||||
max_output: usize,
|
||||
max_lines: usize,
|
||||
stream: Option<StdoutStream>,
|
||||
is_stderr: bool,
|
||||
aggregate_tx: Option<Sender<Vec<u8>>>,
|
||||
) -> io::Result<StreamOutput<Vec<u8>>> {
|
||||
let mut buf = Vec::with_capacity(max_output.min(8 * 1024));
|
||||
let mut tmp = [0u8; 8192];
|
||||
let mut buf = Vec::with_capacity(AGGREGATE_BUFFER_INITIAL_CAPACITY);
|
||||
let mut tmp = [0u8; READ_CHUNK_SIZE];
|
||||
|
||||
let mut remaining_bytes = max_output;
|
||||
let mut remaining_lines = max_lines;
|
||||
// No caps: append all bytes
|
||||
|
||||
loop {
|
||||
let n = reader.read(&mut tmp).await?;
|
||||
@@ -355,33 +372,17 @@ async fn read_capped<R: AsyncRead + Unpin + Send + 'static>(
|
||||
let _ = stream.tx_event.send(event).await;
|
||||
}
|
||||
|
||||
// Copy into the buffer only while we still have byte and line budget.
|
||||
if remaining_bytes > 0 && remaining_lines > 0 {
|
||||
let mut copy_len = 0;
|
||||
for &b in &tmp[..n] {
|
||||
if remaining_bytes == 0 || remaining_lines == 0 {
|
||||
break;
|
||||
}
|
||||
copy_len += 1;
|
||||
remaining_bytes -= 1;
|
||||
if b == b'\n' {
|
||||
remaining_lines -= 1;
|
||||
}
|
||||
}
|
||||
buf.extend_from_slice(&tmp[..copy_len]);
|
||||
if let Some(tx) = &aggregate_tx {
|
||||
let _ = tx.send(tmp[..n].to_vec()).await;
|
||||
}
|
||||
// Continue reading to EOF to avoid back-pressure, but discard once caps are hit.
|
||||
}
|
||||
|
||||
let truncated = remaining_lines == 0 || remaining_bytes == 0;
|
||||
append_all(&mut buf, &tmp[..n]);
|
||||
// Continue reading to EOF to avoid back-pressure
|
||||
}
|
||||
|
||||
Ok(StreamOutput {
|
||||
text: buf,
|
||||
truncated_after_lines: if truncated {
|
||||
Some((max_lines - remaining_lines) as u32)
|
||||
} else {
|
||||
None
|
||||
},
|
||||
truncated_after_lines: None,
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
57
codex-rs/core/src/exec_command/exec_command_params.rs
Normal file
57
codex-rs/core/src/exec_command/exec_command_params.rs
Normal file
@@ -0,0 +1,57 @@
|
||||
use serde::Deserialize;
|
||||
use serde::Serialize;
|
||||
|
||||
use crate::exec_command::session_id::SessionId;
|
||||
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
pub struct ExecCommandParams {
|
||||
pub(crate) cmd: String,
|
||||
|
||||
#[serde(default = "default_yield_time")]
|
||||
pub(crate) yield_time_ms: u64,
|
||||
|
||||
#[serde(default = "max_output_tokens")]
|
||||
pub(crate) max_output_tokens: u64,
|
||||
|
||||
#[serde(default = "default_shell")]
|
||||
pub(crate) shell: String,
|
||||
|
||||
#[serde(default = "default_login")]
|
||||
pub(crate) login: bool,
|
||||
}
|
||||
|
||||
fn default_yield_time() -> u64 {
|
||||
10_000
|
||||
}
|
||||
|
||||
fn max_output_tokens() -> u64 {
|
||||
10_000
|
||||
}
|
||||
|
||||
fn default_login() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
fn default_shell() -> String {
|
||||
"/bin/bash".to_string()
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, Serialize)]
|
||||
pub struct WriteStdinParams {
|
||||
pub(crate) session_id: SessionId,
|
||||
pub(crate) chars: String,
|
||||
|
||||
#[serde(default = "write_stdin_default_yield_time_ms")]
|
||||
pub(crate) yield_time_ms: u64,
|
||||
|
||||
#[serde(default = "write_stdin_default_max_output_tokens")]
|
||||
pub(crate) max_output_tokens: u64,
|
||||
}
|
||||
|
||||
fn write_stdin_default_yield_time_ms() -> u64 {
|
||||
250
|
||||
}
|
||||
|
||||
fn write_stdin_default_max_output_tokens() -> u64 {
|
||||
10_000
|
||||
}
|
||||
83
codex-rs/core/src/exec_command/exec_command_session.rs
Normal file
83
codex-rs/core/src/exec_command/exec_command_session.rs
Normal file
@@ -0,0 +1,83 @@
|
||||
use std::sync::Mutex as StdMutex;
|
||||
|
||||
use tokio::sync::broadcast;
|
||||
use tokio::sync::mpsc;
|
||||
use tokio::task::JoinHandle;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct ExecCommandSession {
|
||||
/// Queue for writing bytes to the process stdin (PTY master write side).
|
||||
writer_tx: mpsc::Sender<Vec<u8>>,
|
||||
/// Broadcast stream of output chunks read from the PTY. New subscribers
|
||||
/// receive only chunks emitted after they subscribe.
|
||||
output_tx: broadcast::Sender<Vec<u8>>,
|
||||
|
||||
/// Child killer handle for termination on drop (can signal independently
|
||||
/// of a thread blocked in `.wait()`).
|
||||
killer: StdMutex<Option<Box<dyn portable_pty::ChildKiller + Send + Sync>>>,
|
||||
|
||||
/// JoinHandle for the blocking PTY reader task.
|
||||
reader_handle: StdMutex<Option<JoinHandle<()>>>,
|
||||
|
||||
/// JoinHandle for the stdin writer task.
|
||||
writer_handle: StdMutex<Option<JoinHandle<()>>>,
|
||||
|
||||
/// JoinHandle for the child wait task.
|
||||
wait_handle: StdMutex<Option<JoinHandle<()>>>,
|
||||
}
|
||||
|
||||
impl ExecCommandSession {
|
||||
pub(crate) fn new(
|
||||
writer_tx: mpsc::Sender<Vec<u8>>,
|
||||
output_tx: broadcast::Sender<Vec<u8>>,
|
||||
killer: Box<dyn portable_pty::ChildKiller + Send + Sync>,
|
||||
reader_handle: JoinHandle<()>,
|
||||
writer_handle: JoinHandle<()>,
|
||||
wait_handle: JoinHandle<()>,
|
||||
) -> Self {
|
||||
Self {
|
||||
writer_tx,
|
||||
output_tx,
|
||||
killer: StdMutex::new(Some(killer)),
|
||||
reader_handle: StdMutex::new(Some(reader_handle)),
|
||||
writer_handle: StdMutex::new(Some(writer_handle)),
|
||||
wait_handle: StdMutex::new(Some(wait_handle)),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn writer_sender(&self) -> mpsc::Sender<Vec<u8>> {
|
||||
self.writer_tx.clone()
|
||||
}
|
||||
|
||||
pub(crate) fn output_receiver(&self) -> broadcast::Receiver<Vec<u8>> {
|
||||
self.output_tx.subscribe()
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for ExecCommandSession {
|
||||
fn drop(&mut self) {
|
||||
// Best-effort: terminate child first so blocking tasks can complete.
|
||||
if let Ok(mut killer_opt) = self.killer.lock()
|
||||
&& let Some(mut killer) = killer_opt.take()
|
||||
{
|
||||
let _ = killer.kill();
|
||||
}
|
||||
|
||||
// Abort background tasks; they may already have exited after kill.
|
||||
if let Ok(mut h) = self.reader_handle.lock()
|
||||
&& let Some(handle) = h.take()
|
||||
{
|
||||
handle.abort();
|
||||
}
|
||||
if let Ok(mut h) = self.writer_handle.lock()
|
||||
&& let Some(handle) = h.take()
|
||||
{
|
||||
handle.abort();
|
||||
}
|
||||
if let Ok(mut h) = self.wait_handle.lock()
|
||||
&& let Some(handle) = h.take()
|
||||
{
|
||||
handle.abort();
|
||||
}
|
||||
}
|
||||
}
|
||||
14
codex-rs/core/src/exec_command/mod.rs
Normal file
14
codex-rs/core/src/exec_command/mod.rs
Normal file
@@ -0,0 +1,14 @@
|
||||
mod exec_command_params;
|
||||
mod exec_command_session;
|
||||
mod responses_api;
|
||||
mod session_id;
|
||||
mod session_manager;
|
||||
|
||||
pub use exec_command_params::ExecCommandParams;
|
||||
pub use exec_command_params::WriteStdinParams;
|
||||
pub use responses_api::EXEC_COMMAND_TOOL_NAME;
|
||||
pub use responses_api::WRITE_STDIN_TOOL_NAME;
|
||||
pub use responses_api::create_exec_command_tool_for_responses_api;
|
||||
pub use responses_api::create_write_stdin_tool_for_responses_api;
|
||||
pub use session_manager::SessionManager as ExecSessionManager;
|
||||
pub use session_manager::result_into_payload;
|
||||
98
codex-rs/core/src/exec_command/responses_api.rs
Normal file
98
codex-rs/core/src/exec_command/responses_api.rs
Normal file
@@ -0,0 +1,98 @@
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use crate::openai_tools::JsonSchema;
|
||||
use crate::openai_tools::ResponsesApiTool;
|
||||
|
||||
pub const EXEC_COMMAND_TOOL_NAME: &str = "exec_command";
|
||||
pub const WRITE_STDIN_TOOL_NAME: &str = "write_stdin";
|
||||
|
||||
pub fn create_exec_command_tool_for_responses_api() -> ResponsesApiTool {
|
||||
let mut properties = BTreeMap::<String, JsonSchema>::new();
|
||||
properties.insert(
|
||||
"cmd".to_string(),
|
||||
JsonSchema::String {
|
||||
description: Some("The shell command to execute.".to_string()),
|
||||
},
|
||||
);
|
||||
properties.insert(
|
||||
"yield_time_ms".to_string(),
|
||||
JsonSchema::Number {
|
||||
description: Some("The maximum time in milliseconds to wait for output.".to_string()),
|
||||
},
|
||||
);
|
||||
properties.insert(
|
||||
"max_output_tokens".to_string(),
|
||||
JsonSchema::Number {
|
||||
description: Some("The maximum number of tokens to output.".to_string()),
|
||||
},
|
||||
);
|
||||
properties.insert(
|
||||
"shell".to_string(),
|
||||
JsonSchema::String {
|
||||
description: Some("The shell to use. Defaults to \"/bin/bash\".".to_string()),
|
||||
},
|
||||
);
|
||||
properties.insert(
|
||||
"login".to_string(),
|
||||
JsonSchema::Boolean {
|
||||
description: Some(
|
||||
"Whether to run the command as a login shell. Defaults to true.".to_string(),
|
||||
),
|
||||
},
|
||||
);
|
||||
|
||||
ResponsesApiTool {
|
||||
name: EXEC_COMMAND_TOOL_NAME.to_owned(),
|
||||
description: r#"Execute shell commands on the local machine with streaming output."#
|
||||
.to_string(),
|
||||
strict: false,
|
||||
parameters: JsonSchema::Object {
|
||||
properties,
|
||||
required: Some(vec!["cmd".to_string()]),
|
||||
additional_properties: Some(false),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
pub fn create_write_stdin_tool_for_responses_api() -> ResponsesApiTool {
|
||||
let mut properties = BTreeMap::<String, JsonSchema>::new();
|
||||
properties.insert(
|
||||
"session_id".to_string(),
|
||||
JsonSchema::Number {
|
||||
description: Some("The ID of the exec_command session.".to_string()),
|
||||
},
|
||||
);
|
||||
properties.insert(
|
||||
"chars".to_string(),
|
||||
JsonSchema::String {
|
||||
description: Some("The characters to write to stdin.".to_string()),
|
||||
},
|
||||
);
|
||||
properties.insert(
|
||||
"yield_time_ms".to_string(),
|
||||
JsonSchema::Number {
|
||||
description: Some(
|
||||
"The maximum time in milliseconds to wait for output after writing.".to_string(),
|
||||
),
|
||||
},
|
||||
);
|
||||
properties.insert(
|
||||
"max_output_tokens".to_string(),
|
||||
JsonSchema::Number {
|
||||
description: Some("The maximum number of tokens to output.".to_string()),
|
||||
},
|
||||
);
|
||||
|
||||
ResponsesApiTool {
|
||||
name: WRITE_STDIN_TOOL_NAME.to_owned(),
|
||||
description: r#"Write characters to an exec session's stdin. Returns all stdout+stderr received within yield_time_ms.
|
||||
Can write control characters (\u0003 for Ctrl-C), or an empty string to just poll stdout+stderr."#
|
||||
.to_string(),
|
||||
strict: false,
|
||||
parameters: JsonSchema::Object {
|
||||
properties,
|
||||
required: Some(vec!["session_id".to_string(), "chars".to_string()]),
|
||||
additional_properties: Some(false),
|
||||
},
|
||||
}
|
||||
}
|
||||
5
codex-rs/core/src/exec_command/session_id.rs
Normal file
5
codex-rs/core/src/exec_command/session_id.rs
Normal file
@@ -0,0 +1,5 @@
|
||||
use serde::Deserialize;
|
||||
use serde::Serialize;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
pub(crate) struct SessionId(pub u32);
|
||||
674
codex-rs/core/src/exec_command/session_manager.rs
Normal file
674
codex-rs/core/src/exec_command/session_manager.rs
Normal file
@@ -0,0 +1,674 @@
|
||||
use std::collections::HashMap;
|
||||
use std::io::ErrorKind;
|
||||
use std::io::Read;
|
||||
use std::sync::Arc;
|
||||
use std::sync::Mutex as StdMutex;
|
||||
use std::sync::atomic::AtomicU32;
|
||||
|
||||
use portable_pty::CommandBuilder;
|
||||
use portable_pty::PtySize;
|
||||
use portable_pty::native_pty_system;
|
||||
use tokio::sync::Mutex;
|
||||
use tokio::sync::mpsc;
|
||||
use tokio::sync::oneshot;
|
||||
use tokio::time::Duration;
|
||||
use tokio::time::Instant;
|
||||
use tokio::time::timeout;
|
||||
|
||||
use crate::exec_command::exec_command_params::ExecCommandParams;
|
||||
use crate::exec_command::exec_command_params::WriteStdinParams;
|
||||
use crate::exec_command::exec_command_session::ExecCommandSession;
|
||||
use crate::exec_command::session_id::SessionId;
|
||||
use codex_protocol::models::FunctionCallOutputPayload;
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub struct SessionManager {
|
||||
next_session_id: AtomicU32,
|
||||
sessions: Mutex<HashMap<SessionId, ExecCommandSession>>,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct ExecCommandOutput {
|
||||
wall_time: Duration,
|
||||
exit_status: ExitStatus,
|
||||
original_token_count: Option<u64>,
|
||||
output: String,
|
||||
}
|
||||
|
||||
impl ExecCommandOutput {
|
||||
fn to_text_output(&self) -> String {
|
||||
let wall_time_secs = self.wall_time.as_secs_f32();
|
||||
let termination_status = match self.exit_status {
|
||||
ExitStatus::Exited(code) => format!("Process exited with code {code}"),
|
||||
ExitStatus::Ongoing(session_id) => {
|
||||
format!("Process running with session ID {}", session_id.0)
|
||||
}
|
||||
};
|
||||
let truncation_status = match self.original_token_count {
|
||||
Some(tokens) => {
|
||||
format!("\nWarning: truncated output (original token count: {tokens})")
|
||||
}
|
||||
None => "".to_string(),
|
||||
};
|
||||
format!(
|
||||
r#"Wall time: {wall_time_secs:.3} seconds
|
||||
{termination_status}{truncation_status}
|
||||
Output:
|
||||
{output}"#,
|
||||
output = self.output
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum ExitStatus {
|
||||
Exited(i32),
|
||||
Ongoing(SessionId),
|
||||
}
|
||||
|
||||
pub fn result_into_payload(result: Result<ExecCommandOutput, String>) -> FunctionCallOutputPayload {
|
||||
match result {
|
||||
Ok(output) => FunctionCallOutputPayload {
|
||||
content: output.to_text_output(),
|
||||
success: Some(true),
|
||||
},
|
||||
Err(err) => FunctionCallOutputPayload {
|
||||
content: err,
|
||||
success: Some(false),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
impl SessionManager {
|
||||
/// Processes the request and is required to send a response via `outgoing`.
|
||||
pub async fn handle_exec_command_request(
|
||||
&self,
|
||||
params: ExecCommandParams,
|
||||
) -> Result<ExecCommandOutput, String> {
|
||||
// Allocate a session id.
|
||||
let session_id = SessionId(
|
||||
self.next_session_id
|
||||
.fetch_add(1, std::sync::atomic::Ordering::SeqCst),
|
||||
);
|
||||
|
||||
let (session, mut exit_rx) =
|
||||
create_exec_command_session(params.clone())
|
||||
.await
|
||||
.map_err(|err| {
|
||||
format!(
|
||||
"failed to create exec command session for session id {}: {err}",
|
||||
session_id.0
|
||||
)
|
||||
})?;
|
||||
|
||||
// Insert into session map.
|
||||
let mut output_rx = session.output_receiver();
|
||||
self.sessions.lock().await.insert(session_id, session);
|
||||
|
||||
// Collect output until either timeout expires or process exits.
|
||||
// Do not cap during collection; truncate at the end if needed.
|
||||
// Use a modest initial capacity to avoid large preallocation.
|
||||
let cap_bytes_u64 = params.max_output_tokens.saturating_mul(4);
|
||||
let cap_bytes: usize = cap_bytes_u64.min(usize::MAX as u64) as usize;
|
||||
let mut collected: Vec<u8> = Vec::with_capacity(4096);
|
||||
|
||||
let start_time = Instant::now();
|
||||
let deadline = start_time + Duration::from_millis(params.yield_time_ms);
|
||||
let mut exit_code: Option<i32> = None;
|
||||
|
||||
loop {
|
||||
if Instant::now() >= deadline {
|
||||
break;
|
||||
}
|
||||
let remaining = deadline.saturating_duration_since(Instant::now());
|
||||
tokio::select! {
|
||||
biased;
|
||||
exit = &mut exit_rx => {
|
||||
exit_code = exit.ok();
|
||||
// Small grace period to pull remaining buffered output
|
||||
let grace_deadline = Instant::now() + Duration::from_millis(25);
|
||||
while Instant::now() < grace_deadline {
|
||||
match timeout(Duration::from_millis(1), output_rx.recv()).await {
|
||||
Ok(Ok(chunk)) => {
|
||||
collected.extend_from_slice(&chunk);
|
||||
}
|
||||
Ok(Err(tokio::sync::broadcast::error::RecvError::Lagged(_))) => {
|
||||
// Skip missed messages; keep trying within grace period.
|
||||
continue;
|
||||
}
|
||||
Ok(Err(tokio::sync::broadcast::error::RecvError::Closed)) => break,
|
||||
Err(_) => break,
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
chunk = timeout(remaining, output_rx.recv()) => {
|
||||
match chunk {
|
||||
Ok(Ok(chunk)) => {
|
||||
collected.extend_from_slice(&chunk);
|
||||
}
|
||||
Ok(Err(tokio::sync::broadcast::error::RecvError::Lagged(_))) => {
|
||||
// Skip missed messages; continue collecting fresh output.
|
||||
}
|
||||
Ok(Err(tokio::sync::broadcast::error::RecvError::Closed)) => { break; }
|
||||
Err(_) => { break; }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let output = String::from_utf8_lossy(&collected).to_string();
|
||||
|
||||
let exit_status = if let Some(code) = exit_code {
|
||||
ExitStatus::Exited(code)
|
||||
} else {
|
||||
ExitStatus::Ongoing(session_id)
|
||||
};
|
||||
|
||||
// If output exceeds cap, truncate the middle and record original token estimate.
|
||||
let (output, original_token_count) = truncate_middle(&output, cap_bytes);
|
||||
Ok(ExecCommandOutput {
|
||||
wall_time: Instant::now().duration_since(start_time),
|
||||
exit_status,
|
||||
original_token_count,
|
||||
output,
|
||||
})
|
||||
}
|
||||
|
||||
/// Write characters to a session's stdin and collect combined output for up to `yield_time_ms`.
|
||||
pub async fn handle_write_stdin_request(
|
||||
&self,
|
||||
params: WriteStdinParams,
|
||||
) -> Result<ExecCommandOutput, String> {
|
||||
let WriteStdinParams {
|
||||
session_id,
|
||||
chars,
|
||||
yield_time_ms,
|
||||
max_output_tokens,
|
||||
} = params;
|
||||
|
||||
// Grab handles without holding the sessions lock across await points.
|
||||
let (writer_tx, mut output_rx) = {
|
||||
let sessions = self.sessions.lock().await;
|
||||
match sessions.get(&session_id) {
|
||||
Some(session) => (session.writer_sender(), session.output_receiver()),
|
||||
None => {
|
||||
return Err(format!("unknown session id {}", session_id.0));
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// Write stdin if provided.
|
||||
if !chars.is_empty() && writer_tx.send(chars.into_bytes()).await.is_err() {
|
||||
return Err("failed to write to stdin".to_string());
|
||||
}
|
||||
|
||||
// Collect output up to yield_time_ms, truncating to max_output_tokens bytes.
|
||||
let mut collected: Vec<u8> = Vec::with_capacity(4096);
|
||||
let start_time = Instant::now();
|
||||
let deadline = start_time + Duration::from_millis(yield_time_ms);
|
||||
loop {
|
||||
let now = Instant::now();
|
||||
if now >= deadline {
|
||||
break;
|
||||
}
|
||||
let remaining = deadline - now;
|
||||
match timeout(remaining, output_rx.recv()).await {
|
||||
Ok(Ok(chunk)) => {
|
||||
// Collect all output within the time budget; truncate at the end.
|
||||
collected.extend_from_slice(&chunk);
|
||||
}
|
||||
Ok(Err(tokio::sync::broadcast::error::RecvError::Lagged(_))) => {
|
||||
// Skip missed messages; continue collecting fresh output.
|
||||
}
|
||||
Ok(Err(tokio::sync::broadcast::error::RecvError::Closed)) => break,
|
||||
Err(_) => break, // timeout
|
||||
}
|
||||
}
|
||||
|
||||
// Return structured output, truncating middle if over cap.
|
||||
let output = String::from_utf8_lossy(&collected).to_string();
|
||||
let cap_bytes_u64 = max_output_tokens.saturating_mul(4);
|
||||
let cap_bytes: usize = cap_bytes_u64.min(usize::MAX as u64) as usize;
|
||||
let (output, original_token_count) = truncate_middle(&output, cap_bytes);
|
||||
Ok(ExecCommandOutput {
|
||||
wall_time: Instant::now().duration_since(start_time),
|
||||
exit_status: ExitStatus::Ongoing(session_id),
|
||||
original_token_count,
|
||||
output,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Spawn PTY and child process per spawn_exec_command_session logic.
|
||||
async fn create_exec_command_session(
|
||||
params: ExecCommandParams,
|
||||
) -> anyhow::Result<(ExecCommandSession, oneshot::Receiver<i32>)> {
|
||||
let ExecCommandParams {
|
||||
cmd,
|
||||
yield_time_ms: _,
|
||||
max_output_tokens: _,
|
||||
shell,
|
||||
login,
|
||||
} = params;
|
||||
|
||||
// Use the native pty implementation for the system
|
||||
let pty_system = native_pty_system();
|
||||
|
||||
// Create a new pty
|
||||
let pair = pty_system.openpty(PtySize {
|
||||
rows: 24,
|
||||
cols: 80,
|
||||
pixel_width: 0,
|
||||
pixel_height: 0,
|
||||
})?;
|
||||
|
||||
// Spawn a shell into the pty
|
||||
let mut command_builder = CommandBuilder::new(shell);
|
||||
let shell_mode_opt = if login { "-lc" } else { "-c" };
|
||||
command_builder.arg(shell_mode_opt);
|
||||
command_builder.arg(cmd);
|
||||
|
||||
let mut child = pair.slave.spawn_command(command_builder)?;
|
||||
// Obtain a killer that can signal the process independently of `.wait()`.
|
||||
let killer = child.clone_killer();
|
||||
|
||||
// Channel to forward write requests to the PTY writer.
|
||||
let (writer_tx, mut writer_rx) = mpsc::channel::<Vec<u8>>(128);
|
||||
// Broadcast for streaming PTY output to readers: subscribers receive from subscription time.
|
||||
let (output_tx, _) = tokio::sync::broadcast::channel::<Vec<u8>>(256);
|
||||
|
||||
// Reader task: drain PTY and forward chunks to output channel.
|
||||
let mut reader = pair.master.try_clone_reader()?;
|
||||
let output_tx_clone = output_tx.clone();
|
||||
let reader_handle = tokio::task::spawn_blocking(move || {
|
||||
let mut buf = [0u8; 8192];
|
||||
loop {
|
||||
match reader.read(&mut buf) {
|
||||
Ok(0) => break, // EOF
|
||||
Ok(n) => {
|
||||
// Forward to broadcast; best-effort if there are subscribers.
|
||||
let _ = output_tx_clone.send(buf[..n].to_vec());
|
||||
}
|
||||
Err(ref e) if e.kind() == ErrorKind::Interrupted => {
|
||||
// Retry on EINTR
|
||||
continue;
|
||||
}
|
||||
Err(ref e) if e.kind() == ErrorKind::WouldBlock => {
|
||||
// We're in a blocking thread; back off briefly and retry.
|
||||
std::thread::sleep(Duration::from_millis(5));
|
||||
continue;
|
||||
}
|
||||
Err(_) => break,
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// Writer task: apply stdin writes to the PTY writer.
|
||||
let writer = pair.master.take_writer()?;
|
||||
let writer = Arc::new(StdMutex::new(writer));
|
||||
let writer_handle = tokio::spawn({
|
||||
let writer = writer.clone();
|
||||
async move {
|
||||
while let Some(bytes) = writer_rx.recv().await {
|
||||
let writer = writer.clone();
|
||||
// Perform blocking write on a blocking thread.
|
||||
let _ = tokio::task::spawn_blocking(move || {
|
||||
if let Ok(mut guard) = writer.lock() {
|
||||
use std::io::Write;
|
||||
let _ = guard.write_all(&bytes);
|
||||
let _ = guard.flush();
|
||||
}
|
||||
})
|
||||
.await;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// Keep the child alive until it exits, then signal exit code.
|
||||
let (exit_tx, exit_rx) = oneshot::channel::<i32>();
|
||||
let wait_handle = tokio::task::spawn_blocking(move || {
|
||||
let code = match child.wait() {
|
||||
Ok(status) => status.exit_code() as i32,
|
||||
Err(_) => -1,
|
||||
};
|
||||
let _ = exit_tx.send(code);
|
||||
});
|
||||
|
||||
// Create and store the session with channels.
|
||||
let session = ExecCommandSession::new(
|
||||
writer_tx,
|
||||
output_tx,
|
||||
killer,
|
||||
reader_handle,
|
||||
writer_handle,
|
||||
wait_handle,
|
||||
);
|
||||
Ok((session, exit_rx))
|
||||
}
|
||||
|
||||
/// Truncate the middle of a UTF-8 string to at most `max_bytes` bytes,
|
||||
/// preserving the beginning and the end. Returns the possibly truncated
|
||||
/// string and `Some(original_token_count)` (estimated at 4 bytes/token)
|
||||
/// if truncation occurred; otherwise returns the original string and `None`.
|
||||
fn truncate_middle(s: &str, max_bytes: usize) -> (String, Option<u64>) {
|
||||
// No truncation needed
|
||||
if s.len() <= max_bytes {
|
||||
return (s.to_string(), None);
|
||||
}
|
||||
let est_tokens = (s.len() as u64).div_ceil(4);
|
||||
if max_bytes == 0 {
|
||||
// Cannot keep any content; still return a full marker (never truncated).
|
||||
return (
|
||||
format!("…{} tokens truncated…", est_tokens),
|
||||
Some(est_tokens),
|
||||
);
|
||||
}
|
||||
|
||||
// Helper to truncate a string to a given byte length on a char boundary.
|
||||
fn truncate_on_boundary(input: &str, max_len: usize) -> &str {
|
||||
if input.len() <= max_len {
|
||||
return input;
|
||||
}
|
||||
let mut end = max_len;
|
||||
while end > 0 && !input.is_char_boundary(end) {
|
||||
end -= 1;
|
||||
}
|
||||
&input[..end]
|
||||
}
|
||||
|
||||
// Given a left/right budget, prefer newline boundaries; otherwise fall back
|
||||
// to UTF-8 char boundaries.
|
||||
fn pick_prefix_end(s: &str, left_budget: usize) -> usize {
|
||||
if let Some(head) = s.get(..left_budget)
|
||||
&& let Some(i) = head.rfind('\n')
|
||||
{
|
||||
return i + 1; // keep the newline so suffix starts on a fresh line
|
||||
}
|
||||
truncate_on_boundary(s, left_budget).len()
|
||||
}
|
||||
|
||||
fn pick_suffix_start(s: &str, right_budget: usize) -> usize {
|
||||
let start_tail = s.len().saturating_sub(right_budget);
|
||||
if let Some(tail) = s.get(start_tail..)
|
||||
&& let Some(i) = tail.find('\n')
|
||||
{
|
||||
return start_tail + i + 1; // start after newline
|
||||
}
|
||||
// Fall back to a char boundary at or after start_tail.
|
||||
let mut idx = start_tail.min(s.len());
|
||||
while idx < s.len() && !s.is_char_boundary(idx) {
|
||||
idx += 1;
|
||||
}
|
||||
idx
|
||||
}
|
||||
|
||||
// Refine marker length and budgets until stable. Marker is never truncated.
|
||||
let mut guess_tokens = est_tokens; // worst-case: everything truncated
|
||||
for _ in 0..4 {
|
||||
let marker = format!("…{} tokens truncated…", guess_tokens);
|
||||
let marker_len = marker.len();
|
||||
let keep_budget = max_bytes.saturating_sub(marker_len);
|
||||
if keep_budget == 0 {
|
||||
// No room for any content within the cap; return a full, untruncated marker
|
||||
// that reflects the entire truncated content.
|
||||
return (
|
||||
format!("…{} tokens truncated…", est_tokens),
|
||||
Some(est_tokens),
|
||||
);
|
||||
}
|
||||
|
||||
let left_budget = keep_budget / 2;
|
||||
let right_budget = keep_budget - left_budget;
|
||||
let prefix_end = pick_prefix_end(s, left_budget);
|
||||
let mut suffix_start = pick_suffix_start(s, right_budget);
|
||||
if suffix_start < prefix_end {
|
||||
suffix_start = prefix_end;
|
||||
}
|
||||
let kept_content_bytes = prefix_end + (s.len() - suffix_start);
|
||||
let truncated_content_bytes = s.len().saturating_sub(kept_content_bytes);
|
||||
let new_tokens = (truncated_content_bytes as u64).div_ceil(4);
|
||||
if new_tokens == guess_tokens {
|
||||
let mut out = String::with_capacity(marker_len + kept_content_bytes + 1);
|
||||
out.push_str(&s[..prefix_end]);
|
||||
out.push_str(&marker);
|
||||
// Place marker on its own line for symmetry when we keep line boundaries.
|
||||
out.push('\n');
|
||||
out.push_str(&s[suffix_start..]);
|
||||
return (out, Some(est_tokens));
|
||||
}
|
||||
guess_tokens = new_tokens;
|
||||
}
|
||||
|
||||
// Fallback: use last guess to build output.
|
||||
let marker = format!("…{} tokens truncated…", guess_tokens);
|
||||
let marker_len = marker.len();
|
||||
let keep_budget = max_bytes.saturating_sub(marker_len);
|
||||
if keep_budget == 0 {
|
||||
return (
|
||||
format!("…{} tokens truncated…", est_tokens),
|
||||
Some(est_tokens),
|
||||
);
|
||||
}
|
||||
let left_budget = keep_budget / 2;
|
||||
let right_budget = keep_budget - left_budget;
|
||||
let prefix_end = pick_prefix_end(s, left_budget);
|
||||
let suffix_start = pick_suffix_start(s, right_budget);
|
||||
let mut out = String::with_capacity(marker_len + prefix_end + (s.len() - suffix_start) + 1);
|
||||
out.push_str(&s[..prefix_end]);
|
||||
out.push_str(&marker);
|
||||
out.push('\n');
|
||||
out.push_str(&s[suffix_start..]);
|
||||
(out, Some(est_tokens))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::exec_command::session_id::SessionId;
|
||||
|
||||
/// Test that verifies that [`SessionManager::handle_exec_command_request()`]
|
||||
/// and [`SessionManager::handle_write_stdin_request()`] work as expected
|
||||
/// in the presence of a process that never terminates (but produces
|
||||
/// output continuously).
|
||||
#[cfg(unix)]
|
||||
#[allow(clippy::print_stderr)]
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn session_manager_streams_and_truncates_from_now() {
|
||||
use crate::exec_command::exec_command_params::ExecCommandParams;
|
||||
use crate::exec_command::exec_command_params::WriteStdinParams;
|
||||
use tokio::time::sleep;
|
||||
|
||||
let session_manager = SessionManager::default();
|
||||
// Long-running loop that prints an increasing counter every ~100ms.
|
||||
// Use Python for a portable, reliable sleep across shells/PTYs.
|
||||
let cmd = r#"python3 - <<'PY'
|
||||
import sys, time
|
||||
count = 0
|
||||
while True:
|
||||
print(count)
|
||||
sys.stdout.flush()
|
||||
count += 100
|
||||
time.sleep(0.1)
|
||||
PY"#
|
||||
.to_string();
|
||||
|
||||
// Start the session and collect ~3s of output.
|
||||
let params = ExecCommandParams {
|
||||
cmd,
|
||||
yield_time_ms: 3_000,
|
||||
max_output_tokens: 1_000, // large enough to avoid truncation here
|
||||
shell: "/bin/bash".to_string(),
|
||||
login: false,
|
||||
};
|
||||
let initial_output = match session_manager
|
||||
.handle_exec_command_request(params.clone())
|
||||
.await
|
||||
{
|
||||
Ok(v) => v,
|
||||
Err(e) => {
|
||||
// PTY may be restricted in some sandboxes; skip in that case.
|
||||
if e.contains("openpty") || e.contains("Operation not permitted") {
|
||||
eprintln!("skipping test due to restricted PTY: {e}");
|
||||
return;
|
||||
}
|
||||
panic!("exec request failed unexpectedly: {e}");
|
||||
}
|
||||
};
|
||||
eprintln!("initial output: {initial_output:?}");
|
||||
|
||||
// Should be ongoing (we launched a never-ending loop).
|
||||
let session_id = match initial_output.exit_status {
|
||||
ExitStatus::Ongoing(id) => id,
|
||||
_ => panic!("expected ongoing session"),
|
||||
};
|
||||
|
||||
// Parse the numeric lines and get the max observed value in the first window.
|
||||
let first_nums = extract_monotonic_numbers(&initial_output.output);
|
||||
assert!(
|
||||
!first_nums.is_empty(),
|
||||
"expected some output from first window"
|
||||
);
|
||||
let first_max = *first_nums.iter().max().unwrap();
|
||||
|
||||
// Wait ~4s so counters progress while we're not reading.
|
||||
sleep(Duration::from_millis(4_000)).await;
|
||||
|
||||
// Now read ~3s of output "from now" only.
|
||||
// Use a small token cap so truncation occurs and we test middle truncation.
|
||||
let write_params = WriteStdinParams {
|
||||
session_id,
|
||||
chars: String::new(),
|
||||
yield_time_ms: 3_000,
|
||||
max_output_tokens: 16, // 16 tokens ~= 64 bytes -> likely truncation
|
||||
};
|
||||
let second = session_manager
|
||||
.handle_write_stdin_request(write_params)
|
||||
.await
|
||||
.expect("write stdin should succeed");
|
||||
|
||||
// Verify truncation metadata and size bound (cap is tokens*4 bytes).
|
||||
assert!(second.original_token_count.is_some());
|
||||
let cap_bytes = (16u64 * 4) as usize;
|
||||
assert!(second.output.len() <= cap_bytes);
|
||||
// New middle marker should be present.
|
||||
assert!(
|
||||
second.output.contains("tokens truncated") && second.output.contains('…'),
|
||||
"expected truncation marker in output, got: {}",
|
||||
second.output
|
||||
);
|
||||
|
||||
// Minimal freshness check: the earliest number we see in the second window
|
||||
// should be significantly larger than the last from the first window.
|
||||
let second_nums = extract_monotonic_numbers(&second.output);
|
||||
assert!(
|
||||
!second_nums.is_empty(),
|
||||
"expected some numeric output from second window"
|
||||
);
|
||||
let second_min = *second_nums.iter().min().unwrap();
|
||||
|
||||
// We slept 4 seconds (~40 ticks at 100ms/tick, each +100), so expect
|
||||
// an increase of roughly 4000 or more. Allow a generous margin.
|
||||
assert!(
|
||||
second_min >= first_max + 2000,
|
||||
"second_min={second_min} first_max={first_max}",
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
fn extract_monotonic_numbers(s: &str) -> Vec<i64> {
|
||||
s.lines()
|
||||
.filter_map(|line| {
|
||||
if !line.is_empty()
|
||||
&& line.chars().all(|c| c.is_ascii_digit())
|
||||
&& let Ok(n) = line.parse::<i64>()
|
||||
{
|
||||
// Our generator increments by 100; ignore spurious fragments.
|
||||
if n % 100 == 0 {
|
||||
return Some(n);
|
||||
}
|
||||
}
|
||||
None
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn to_text_output_exited_no_truncation() {
|
||||
let out = ExecCommandOutput {
|
||||
wall_time: Duration::from_millis(1234),
|
||||
exit_status: ExitStatus::Exited(0),
|
||||
original_token_count: None,
|
||||
output: "hello".to_string(),
|
||||
};
|
||||
let text = out.to_text_output();
|
||||
let expected = r#"Wall time: 1.234 seconds
|
||||
Process exited with code 0
|
||||
Output:
|
||||
hello"#;
|
||||
assert_eq!(expected, text);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn to_text_output_ongoing_with_truncation() {
|
||||
let out = ExecCommandOutput {
|
||||
wall_time: Duration::from_millis(500),
|
||||
exit_status: ExitStatus::Ongoing(SessionId(42)),
|
||||
original_token_count: Some(1000),
|
||||
output: "abc".to_string(),
|
||||
};
|
||||
let text = out.to_text_output();
|
||||
let expected = r#"Wall time: 0.500 seconds
|
||||
Process running with session ID 42
|
||||
Warning: truncated output (original token count: 1000)
|
||||
Output:
|
||||
abc"#;
|
||||
assert_eq!(expected, text);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn truncate_middle_no_newlines_fallback() {
|
||||
// A long string with no newlines that exceeds the cap.
|
||||
let s = "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ";
|
||||
let max_bytes = 16; // force truncation
|
||||
let (out, original) = truncate_middle(s, max_bytes);
|
||||
// For very small caps, we return the full, untruncated marker,
|
||||
// even if it exceeds the cap.
|
||||
assert_eq!(out, "…16 tokens truncated…");
|
||||
// Original string length is 62 bytes => ceil(62/4) = 16 tokens.
|
||||
assert_eq!(original, Some(16));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn truncate_middle_prefers_newline_boundaries() {
|
||||
// Build a multi-line string of 20 numbered lines (each "NNN\n").
|
||||
let mut s = String::new();
|
||||
for i in 1..=20 {
|
||||
s.push_str(&format!("{i:03}\n"));
|
||||
}
|
||||
// Total length: 20 lines * 4 bytes per line = 80 bytes.
|
||||
assert_eq!(s.len(), 80);
|
||||
|
||||
// Choose a cap that forces truncation while leaving room for
|
||||
// a few lines on each side after accounting for the marker.
|
||||
let max_bytes = 64;
|
||||
// Expect exact output: first 4 lines, marker, last 4 lines, and correct token estimate (80/4 = 20).
|
||||
assert_eq!(
|
||||
truncate_middle(&s, max_bytes),
|
||||
(
|
||||
r#"001
|
||||
002
|
||||
003
|
||||
004
|
||||
…12 tokens truncated…
|
||||
017
|
||||
018
|
||||
019
|
||||
020
|
||||
"#
|
||||
.to_string(),
|
||||
Some(20)
|
||||
)
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -20,6 +20,7 @@ mod conversation_history;
|
||||
mod environment_context;
|
||||
pub mod error;
|
||||
pub mod exec;
|
||||
mod exec_command;
|
||||
pub mod exec_env;
|
||||
mod flags;
|
||||
pub mod git_info;
|
||||
|
||||
@@ -90,7 +90,6 @@ pub fn find_family_for_model(slug: &str) -> Option<ModelFamily> {
|
||||
model_family!(
|
||||
slug, slug,
|
||||
supports_reasoning_summaries: true,
|
||||
apply_patch_tool_type: Some(ApplyPatchToolType::Freeform),
|
||||
)
|
||||
} else if slug.starts_with("gpt-4.1") {
|
||||
model_family!(
|
||||
@@ -107,7 +106,6 @@ pub fn find_family_for_model(slug: &str) -> Option<ModelFamily> {
|
||||
model_family!(
|
||||
slug, "gpt-5",
|
||||
supports_reasoning_summaries: true,
|
||||
apply_patch_tool_type: Some(ApplyPatchToolType::Freeform),
|
||||
)
|
||||
} else {
|
||||
None
|
||||
|
||||
@@ -79,13 +79,13 @@ pub(crate) fn get_model_info(model_family: &ModelFamily) -> Option<ModelInfo> {
|
||||
}),
|
||||
|
||||
"gpt-5" => Some(ModelInfo {
|
||||
context_window: 200_000,
|
||||
max_output_tokens: 100_000,
|
||||
context_window: 400_000,
|
||||
max_output_tokens: 128_000,
|
||||
}),
|
||||
|
||||
_ if slug.starts_with("codex-") => Some(ModelInfo {
|
||||
context_window: 200_000,
|
||||
max_output_tokens: 100_000,
|
||||
context_window: 400_000,
|
||||
max_output_tokens: 128_000,
|
||||
}),
|
||||
|
||||
_ => None,
|
||||
|
||||
@@ -47,6 +47,8 @@ pub(crate) enum OpenAiTool {
|
||||
Function(ResponsesApiTool),
|
||||
#[serde(rename = "local_shell")]
|
||||
LocalShell {},
|
||||
#[serde(rename = "web_search")]
|
||||
WebSearch {},
|
||||
#[serde(rename = "custom")]
|
||||
Freeform(FreeformTool),
|
||||
}
|
||||
@@ -56,29 +58,46 @@ pub enum ConfigShellToolType {
|
||||
DefaultShell,
|
||||
ShellWithRequest { sandbox_policy: SandboxPolicy },
|
||||
LocalShell,
|
||||
StreamableShell,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ToolsConfig {
|
||||
pub(crate) struct ToolsConfig {
|
||||
pub shell_type: ConfigShellToolType,
|
||||
pub plan_tool: bool,
|
||||
pub apply_patch_tool_type: Option<ApplyPatchToolType>,
|
||||
pub web_search_request: bool,
|
||||
}
|
||||
|
||||
pub(crate) struct ToolsConfigParams<'a> {
|
||||
pub(crate) model_family: &'a ModelFamily,
|
||||
pub(crate) approval_policy: AskForApproval,
|
||||
pub(crate) sandbox_policy: SandboxPolicy,
|
||||
pub(crate) include_plan_tool: bool,
|
||||
pub(crate) include_apply_patch_tool: bool,
|
||||
pub(crate) include_web_search_request: bool,
|
||||
pub(crate) use_streamable_shell_tool: bool,
|
||||
}
|
||||
|
||||
impl ToolsConfig {
|
||||
pub fn new(
|
||||
model_family: &ModelFamily,
|
||||
approval_policy: AskForApproval,
|
||||
sandbox_policy: SandboxPolicy,
|
||||
include_plan_tool: bool,
|
||||
include_apply_patch_tool: bool,
|
||||
) -> Self {
|
||||
let mut shell_type = if model_family.uses_local_shell_tool {
|
||||
pub fn new(params: &ToolsConfigParams) -> Self {
|
||||
let ToolsConfigParams {
|
||||
model_family,
|
||||
approval_policy,
|
||||
sandbox_policy,
|
||||
include_plan_tool,
|
||||
include_apply_patch_tool,
|
||||
include_web_search_request,
|
||||
use_streamable_shell_tool,
|
||||
} = params;
|
||||
let mut shell_type = if *use_streamable_shell_tool {
|
||||
ConfigShellToolType::StreamableShell
|
||||
} else if model_family.uses_local_shell_tool {
|
||||
ConfigShellToolType::LocalShell
|
||||
} else {
|
||||
ConfigShellToolType::DefaultShell
|
||||
};
|
||||
if matches!(approval_policy, AskForApproval::OnRequest) {
|
||||
if matches!(approval_policy, AskForApproval::OnRequest) && !use_streamable_shell_tool {
|
||||
shell_type = ConfigShellToolType::ShellWithRequest {
|
||||
sandbox_policy: sandbox_policy.clone(),
|
||||
}
|
||||
@@ -88,7 +107,7 @@ impl ToolsConfig {
|
||||
Some(ApplyPatchToolType::Freeform) => Some(ApplyPatchToolType::Freeform),
|
||||
Some(ApplyPatchToolType::Function) => Some(ApplyPatchToolType::Function),
|
||||
None => {
|
||||
if include_apply_patch_tool {
|
||||
if *include_apply_patch_tool {
|
||||
Some(ApplyPatchToolType::Freeform)
|
||||
} else {
|
||||
None
|
||||
@@ -98,8 +117,9 @@ impl ToolsConfig {
|
||||
|
||||
Self {
|
||||
shell_type,
|
||||
plan_tool: include_plan_tool,
|
||||
plan_tool: *include_plan_tool,
|
||||
apply_patch_tool_type,
|
||||
web_search_request: *include_web_search_request,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -492,6 +512,14 @@ pub(crate) fn get_openai_tools(
|
||||
ConfigShellToolType::LocalShell => {
|
||||
tools.push(OpenAiTool::LocalShell {});
|
||||
}
|
||||
ConfigShellToolType::StreamableShell => {
|
||||
tools.push(OpenAiTool::Function(
|
||||
crate::exec_command::create_exec_command_tool_for_responses_api(),
|
||||
));
|
||||
tools.push(OpenAiTool::Function(
|
||||
crate::exec_command::create_write_stdin_tool_for_responses_api(),
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
if config.plan_tool {
|
||||
@@ -509,8 +537,17 @@ pub(crate) fn get_openai_tools(
|
||||
}
|
||||
}
|
||||
|
||||
if config.web_search_request {
|
||||
tools.push(OpenAiTool::WebSearch {});
|
||||
}
|
||||
|
||||
if let Some(mcp_tools) = mcp_tools {
|
||||
for (name, tool) in mcp_tools {
|
||||
// Ensure deterministic ordering to maximize prompt cache hits.
|
||||
// HashMap iteration order is non-deterministic, so sort by fully-qualified tool name.
|
||||
let mut entries: Vec<(String, mcp_types::Tool)> = mcp_tools.into_iter().collect();
|
||||
entries.sort_by(|a, b| a.0.cmp(&b.0));
|
||||
|
||||
for (name, tool) in entries.into_iter() {
|
||||
match mcp_tool_to_openai_tool(name.clone(), tool.clone()) {
|
||||
Ok(converted_tool) => tools.push(OpenAiTool::Function(converted_tool)),
|
||||
Err(e) => {
|
||||
@@ -537,6 +574,7 @@ mod tests {
|
||||
.map(|tool| match tool {
|
||||
OpenAiTool::Function(ResponsesApiTool { name, .. }) => name,
|
||||
OpenAiTool::LocalShell {} => "local_shell",
|
||||
OpenAiTool::WebSearch {} => "web_search",
|
||||
OpenAiTool::Freeform(FreeformTool { name, .. }) => name,
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
@@ -558,43 +596,49 @@ mod tests {
|
||||
fn test_get_openai_tools() {
|
||||
let model_family = find_family_for_model("codex-mini-latest")
|
||||
.expect("codex-mini-latest should be a valid model family");
|
||||
let config = ToolsConfig::new(
|
||||
&model_family,
|
||||
AskForApproval::Never,
|
||||
SandboxPolicy::ReadOnly,
|
||||
true,
|
||||
false,
|
||||
);
|
||||
let config = ToolsConfig::new(&ToolsConfigParams {
|
||||
model_family: &model_family,
|
||||
approval_policy: AskForApproval::Never,
|
||||
sandbox_policy: SandboxPolicy::ReadOnly,
|
||||
include_plan_tool: true,
|
||||
include_apply_patch_tool: false,
|
||||
include_web_search_request: true,
|
||||
use_streamable_shell_tool: false,
|
||||
});
|
||||
let tools = get_openai_tools(&config, Some(HashMap::new()));
|
||||
|
||||
assert_eq_tool_names(&tools, &["local_shell", "update_plan"]);
|
||||
assert_eq_tool_names(&tools, &["local_shell", "update_plan", "web_search"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_get_openai_tools_default_shell() {
|
||||
let model_family = find_family_for_model("o3").expect("o3 should be a valid model family");
|
||||
let config = ToolsConfig::new(
|
||||
&model_family,
|
||||
AskForApproval::Never,
|
||||
SandboxPolicy::ReadOnly,
|
||||
true,
|
||||
false,
|
||||
);
|
||||
let config = ToolsConfig::new(&ToolsConfigParams {
|
||||
model_family: &model_family,
|
||||
approval_policy: AskForApproval::Never,
|
||||
sandbox_policy: SandboxPolicy::ReadOnly,
|
||||
include_plan_tool: true,
|
||||
include_apply_patch_tool: false,
|
||||
include_web_search_request: true,
|
||||
use_streamable_shell_tool: false,
|
||||
});
|
||||
let tools = get_openai_tools(&config, Some(HashMap::new()));
|
||||
|
||||
assert_eq_tool_names(&tools, &["shell", "update_plan"]);
|
||||
assert_eq_tool_names(&tools, &["shell", "update_plan", "web_search"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_get_openai_tools_mcp_tools() {
|
||||
let model_family = find_family_for_model("o3").expect("o3 should be a valid model family");
|
||||
let config = ToolsConfig::new(
|
||||
&model_family,
|
||||
AskForApproval::Never,
|
||||
SandboxPolicy::ReadOnly,
|
||||
false,
|
||||
false,
|
||||
);
|
||||
let config = ToolsConfig::new(&ToolsConfigParams {
|
||||
model_family: &model_family,
|
||||
approval_policy: AskForApproval::Never,
|
||||
sandbox_policy: SandboxPolicy::ReadOnly,
|
||||
include_plan_tool: false,
|
||||
include_apply_patch_tool: false,
|
||||
include_web_search_request: true,
|
||||
use_streamable_shell_tool: false,
|
||||
});
|
||||
let tools = get_openai_tools(
|
||||
&config,
|
||||
Some(HashMap::from([(
|
||||
@@ -616,8 +660,8 @@ mod tests {
|
||||
"number_property": { "type": "number" },
|
||||
},
|
||||
"required": [
|
||||
"string_property",
|
||||
"number_property"
|
||||
"string_property".to_string(),
|
||||
"number_property".to_string()
|
||||
],
|
||||
"additionalProperties": Some(false),
|
||||
},
|
||||
@@ -633,10 +677,13 @@ mod tests {
|
||||
)])),
|
||||
);
|
||||
|
||||
assert_eq_tool_names(&tools, &["shell", "test_server/do_something_cool"]);
|
||||
assert_eq_tool_names(
|
||||
&tools,
|
||||
&["shell", "web_search", "test_server/do_something_cool"],
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
tools[1],
|
||||
tools[2],
|
||||
OpenAiTool::Function(ResponsesApiTool {
|
||||
name: "test_server/do_something_cool".to_string(),
|
||||
parameters: JsonSchema::Object {
|
||||
@@ -679,16 +726,93 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_get_openai_tools_mcp_tools_sorted_by_name() {
|
||||
let model_family = find_family_for_model("o3").expect("o3 should be a valid model family");
|
||||
let config = ToolsConfig::new(&ToolsConfigParams {
|
||||
model_family: &model_family,
|
||||
approval_policy: AskForApproval::Never,
|
||||
sandbox_policy: SandboxPolicy::ReadOnly,
|
||||
include_plan_tool: false,
|
||||
include_apply_patch_tool: false,
|
||||
include_web_search_request: false,
|
||||
use_streamable_shell_tool: false,
|
||||
});
|
||||
|
||||
// Intentionally construct a map with keys that would sort alphabetically.
|
||||
let tools_map: HashMap<String, mcp_types::Tool> = HashMap::from([
|
||||
(
|
||||
"test_server/do".to_string(),
|
||||
mcp_types::Tool {
|
||||
name: "a".to_string(),
|
||||
input_schema: ToolInputSchema {
|
||||
properties: Some(serde_json::json!({})),
|
||||
required: None,
|
||||
r#type: "object".to_string(),
|
||||
},
|
||||
output_schema: None,
|
||||
title: None,
|
||||
annotations: None,
|
||||
description: Some("a".to_string()),
|
||||
},
|
||||
),
|
||||
(
|
||||
"test_server/something".to_string(),
|
||||
mcp_types::Tool {
|
||||
name: "b".to_string(),
|
||||
input_schema: ToolInputSchema {
|
||||
properties: Some(serde_json::json!({})),
|
||||
required: None,
|
||||
r#type: "object".to_string(),
|
||||
},
|
||||
output_schema: None,
|
||||
title: None,
|
||||
annotations: None,
|
||||
description: Some("b".to_string()),
|
||||
},
|
||||
),
|
||||
(
|
||||
"test_server/cool".to_string(),
|
||||
mcp_types::Tool {
|
||||
name: "c".to_string(),
|
||||
input_schema: ToolInputSchema {
|
||||
properties: Some(serde_json::json!({})),
|
||||
required: None,
|
||||
r#type: "object".to_string(),
|
||||
},
|
||||
output_schema: None,
|
||||
title: None,
|
||||
annotations: None,
|
||||
description: Some("c".to_string()),
|
||||
},
|
||||
),
|
||||
]);
|
||||
|
||||
let tools = get_openai_tools(&config, Some(tools_map));
|
||||
// Expect shell first, followed by MCP tools sorted by fully-qualified name.
|
||||
assert_eq_tool_names(
|
||||
&tools,
|
||||
&[
|
||||
"shell",
|
||||
"test_server/cool",
|
||||
"test_server/do",
|
||||
"test_server/something",
|
||||
],
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_mcp_tool_property_missing_type_defaults_to_string() {
|
||||
let model_family = find_family_for_model("o3").expect("o3 should be a valid model family");
|
||||
let config = ToolsConfig::new(
|
||||
&model_family,
|
||||
AskForApproval::Never,
|
||||
SandboxPolicy::ReadOnly,
|
||||
false,
|
||||
false,
|
||||
);
|
||||
let config = ToolsConfig::new(&ToolsConfigParams {
|
||||
model_family: &model_family,
|
||||
approval_policy: AskForApproval::Never,
|
||||
sandbox_policy: SandboxPolicy::ReadOnly,
|
||||
include_plan_tool: false,
|
||||
include_apply_patch_tool: false,
|
||||
include_web_search_request: true,
|
||||
use_streamable_shell_tool: false,
|
||||
});
|
||||
|
||||
let tools = get_openai_tools(
|
||||
&config,
|
||||
@@ -713,10 +837,10 @@ mod tests {
|
||||
)])),
|
||||
);
|
||||
|
||||
assert_eq_tool_names(&tools, &["shell", "dash/search"]);
|
||||
assert_eq_tool_names(&tools, &["shell", "web_search", "dash/search"]);
|
||||
|
||||
assert_eq!(
|
||||
tools[1],
|
||||
tools[2],
|
||||
OpenAiTool::Function(ResponsesApiTool {
|
||||
name: "dash/search".to_string(),
|
||||
parameters: JsonSchema::Object {
|
||||
@@ -738,13 +862,15 @@ mod tests {
|
||||
#[test]
|
||||
fn test_mcp_tool_integer_normalized_to_number() {
|
||||
let model_family = find_family_for_model("o3").expect("o3 should be a valid model family");
|
||||
let config = ToolsConfig::new(
|
||||
&model_family,
|
||||
AskForApproval::Never,
|
||||
SandboxPolicy::ReadOnly,
|
||||
false,
|
||||
false,
|
||||
);
|
||||
let config = ToolsConfig::new(&ToolsConfigParams {
|
||||
model_family: &model_family,
|
||||
approval_policy: AskForApproval::Never,
|
||||
sandbox_policy: SandboxPolicy::ReadOnly,
|
||||
include_plan_tool: false,
|
||||
include_apply_patch_tool: false,
|
||||
include_web_search_request: true,
|
||||
use_streamable_shell_tool: false,
|
||||
});
|
||||
|
||||
let tools = get_openai_tools(
|
||||
&config,
|
||||
@@ -767,9 +893,9 @@ mod tests {
|
||||
)])),
|
||||
);
|
||||
|
||||
assert_eq_tool_names(&tools, &["shell", "dash/paginate"]);
|
||||
assert_eq_tool_names(&tools, &["shell", "web_search", "dash/paginate"]);
|
||||
assert_eq!(
|
||||
tools[1],
|
||||
tools[2],
|
||||
OpenAiTool::Function(ResponsesApiTool {
|
||||
name: "dash/paginate".to_string(),
|
||||
parameters: JsonSchema::Object {
|
||||
@@ -789,13 +915,15 @@ mod tests {
|
||||
#[test]
|
||||
fn test_mcp_tool_array_without_items_gets_default_string_items() {
|
||||
let model_family = find_family_for_model("o3").expect("o3 should be a valid model family");
|
||||
let config = ToolsConfig::new(
|
||||
&model_family,
|
||||
AskForApproval::Never,
|
||||
SandboxPolicy::ReadOnly,
|
||||
false,
|
||||
false,
|
||||
);
|
||||
let config = ToolsConfig::new(&ToolsConfigParams {
|
||||
model_family: &model_family,
|
||||
approval_policy: AskForApproval::Never,
|
||||
sandbox_policy: SandboxPolicy::ReadOnly,
|
||||
include_plan_tool: false,
|
||||
include_apply_patch_tool: false,
|
||||
include_web_search_request: true,
|
||||
use_streamable_shell_tool: false,
|
||||
});
|
||||
|
||||
let tools = get_openai_tools(
|
||||
&config,
|
||||
@@ -818,9 +946,9 @@ mod tests {
|
||||
)])),
|
||||
);
|
||||
|
||||
assert_eq_tool_names(&tools, &["shell", "dash/tags"]);
|
||||
assert_eq_tool_names(&tools, &["shell", "web_search", "dash/tags"]);
|
||||
assert_eq!(
|
||||
tools[1],
|
||||
tools[2],
|
||||
OpenAiTool::Function(ResponsesApiTool {
|
||||
name: "dash/tags".to_string(),
|
||||
parameters: JsonSchema::Object {
|
||||
@@ -843,13 +971,15 @@ mod tests {
|
||||
#[test]
|
||||
fn test_mcp_tool_anyof_defaults_to_string() {
|
||||
let model_family = find_family_for_model("o3").expect("o3 should be a valid model family");
|
||||
let config = ToolsConfig::new(
|
||||
&model_family,
|
||||
AskForApproval::Never,
|
||||
SandboxPolicy::ReadOnly,
|
||||
false,
|
||||
false,
|
||||
);
|
||||
let config = ToolsConfig::new(&ToolsConfigParams {
|
||||
model_family: &model_family,
|
||||
approval_policy: AskForApproval::Never,
|
||||
sandbox_policy: SandboxPolicy::ReadOnly,
|
||||
include_plan_tool: false,
|
||||
include_apply_patch_tool: false,
|
||||
include_web_search_request: true,
|
||||
use_streamable_shell_tool: false,
|
||||
});
|
||||
|
||||
let tools = get_openai_tools(
|
||||
&config,
|
||||
@@ -872,9 +1002,9 @@ mod tests {
|
||||
)])),
|
||||
);
|
||||
|
||||
assert_eq_tool_names(&tools, &["shell", "dash/value"]);
|
||||
assert_eq_tool_names(&tools, &["shell", "web_search", "dash/value"]);
|
||||
assert_eq!(
|
||||
tools[1],
|
||||
tools[2],
|
||||
OpenAiTool::Function(ResponsesApiTool {
|
||||
name: "dash/value".to_string(),
|
||||
parameters: JsonSchema::Object {
|
||||
|
||||
3
codex-rs/core/tests/all.rs
Normal file
3
codex-rs/core/tests/all.rs
Normal file
@@ -0,0 +1,3 @@
|
||||
// Single integration test binary that aggregates all test modules.
|
||||
// The submodules live in `tests/all/`.
|
||||
mod suite;
|
||||
@@ -70,12 +70,12 @@ async fn truncates_output_lines() {
|
||||
|
||||
let output = run_test_cmd(tmp, cmd).await.unwrap();
|
||||
|
||||
let expected_output = (1..=256)
|
||||
let expected_output = (1..=300)
|
||||
.map(|i| format!("{i}\n"))
|
||||
.collect::<Vec<_>>()
|
||||
.join("");
|
||||
assert_eq!(output.stdout.text, expected_output);
|
||||
assert_eq!(output.stdout.truncated_after_lines, Some(256));
|
||||
assert_eq!(output.stdout.truncated_after_lines, None);
|
||||
}
|
||||
|
||||
/// Command succeeds with exit code 0 normally
|
||||
@@ -91,8 +91,8 @@ async fn truncates_output_bytes() {
|
||||
|
||||
let output = run_test_cmd(tmp, cmd).await.unwrap();
|
||||
|
||||
assert_eq!(output.stdout.text.len(), 10240);
|
||||
assert_eq!(output.stdout.truncated_after_lines, Some(10));
|
||||
assert!(output.stdout.text.len() >= 15000);
|
||||
assert_eq!(output.stdout.truncated_after_lines, None);
|
||||
}
|
||||
|
||||
/// Command not found returns exit code 127, this is not considered a sandbox error
|
||||
@@ -139,3 +139,34 @@ async fn test_exec_stderr_stream_events_echo() {
|
||||
}
|
||||
assert_eq!(String::from_utf8_lossy(&err), "oops\n");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_aggregated_output_interleaves_in_order() {
|
||||
// Spawn a shell that alternates stdout and stderr with sleeps to enforce order.
|
||||
let cmd = vec![
|
||||
"/bin/sh".to_string(),
|
||||
"-c".to_string(),
|
||||
"printf 'O1\\n'; sleep 0.01; printf 'E1\\n' 1>&2; sleep 0.01; printf 'O2\\n'; sleep 0.01; printf 'E2\\n' 1>&2".to_string(),
|
||||
];
|
||||
|
||||
let params = ExecParams {
|
||||
command: cmd,
|
||||
cwd: std::env::current_dir().unwrap_or_else(|_| PathBuf::from(".")),
|
||||
timeout_ms: Some(5_000),
|
||||
env: HashMap::new(),
|
||||
with_escalated_permissions: None,
|
||||
justification: None,
|
||||
};
|
||||
|
||||
let policy = SandboxPolicy::new_read_only_policy();
|
||||
|
||||
let result = process_exec_tool_call(params, SandboxType::None, &policy, &None, None)
|
||||
.await
|
||||
.expect("process_exec_tool_call");
|
||||
|
||||
assert_eq!(result.exit_code, 0);
|
||||
assert_eq!(result.stdout.text, "O1\nO2\n");
|
||||
assert_eq!(result.stderr.text, "E1\nE2\n");
|
||||
assert_eq!(result.aggregated_output.text, "O1\nE1\nO2\nE2\n");
|
||||
assert_eq!(result.aggregated_output.truncated_after_lines, None);
|
||||
}
|
||||
12
codex-rs/core/tests/suite/mod.rs
Normal file
12
codex-rs/core/tests/suite/mod.rs
Normal file
@@ -0,0 +1,12 @@
|
||||
// Aggregates all former standalone integration tests as modules.
|
||||
|
||||
mod cli_stream;
|
||||
mod client;
|
||||
mod compact;
|
||||
mod exec;
|
||||
mod exec_stream_events;
|
||||
mod live_cli;
|
||||
mod prompt_caching;
|
||||
mod seatbelt;
|
||||
mod stream_error_allows_next_turn;
|
||||
mod stream_no_completed;
|
||||
@@ -107,8 +107,8 @@ async fn codex_mini_latest_tools() {
|
||||
assert_eq!(requests.len(), 2, "expected two POST requests");
|
||||
|
||||
let expected_instructions = [
|
||||
include_str!("../prompt.md"),
|
||||
include_str!("../../apply-patch/apply_patch_tool_instructions.md"),
|
||||
include_str!("../../prompt.md"),
|
||||
include_str!("../../../apply-patch/apply_patch_tool_instructions.md"),
|
||||
]
|
||||
.join("\n");
|
||||
|
||||
@@ -188,7 +188,7 @@ async fn prompt_tools_are_consistent_across_requests() {
|
||||
let requests = server.received_requests().await.unwrap();
|
||||
assert_eq!(requests.len(), 2, "expected two POST requests");
|
||||
|
||||
let expected_instructions: &str = include_str!("../prompt.md");
|
||||
let expected_instructions: &str = include_str!("../../prompt.md");
|
||||
// our internal implementation is responsible for keeping tools in sync
|
||||
// with the OpenAI schema, so we just verify the tool presence here
|
||||
let expected_tools_names: &[&str] = &["shell", "update_plan", "apply_patch"];
|
||||
@@ -24,6 +24,7 @@ use codex_core::protocol::StreamErrorEvent;
|
||||
use codex_core::protocol::TaskCompleteEvent;
|
||||
use codex_core::protocol::TurnAbortReason;
|
||||
use codex_core::protocol::TurnDiffEvent;
|
||||
use codex_core::protocol::WebSearchBeginEvent;
|
||||
use owo_colors::OwoColorize;
|
||||
use owo_colors::Style;
|
||||
use shlex::try_join;
|
||||
@@ -287,8 +288,7 @@ impl EventProcessor for EventProcessorWithHumanOutput {
|
||||
EventMsg::ExecCommandOutputDelta(_) => {}
|
||||
EventMsg::ExecCommandEnd(ExecCommandEndEvent {
|
||||
call_id,
|
||||
stdout,
|
||||
stderr,
|
||||
aggregated_output,
|
||||
duration,
|
||||
exit_code,
|
||||
..
|
||||
@@ -304,8 +304,7 @@ impl EventProcessor for EventProcessorWithHumanOutput {
|
||||
("".to_string(), format!("exec('{call_id}')"))
|
||||
};
|
||||
|
||||
let output = if exit_code == 0 { stdout } else { stderr };
|
||||
let truncated_output = output
|
||||
let truncated_output = aggregated_output
|
||||
.lines()
|
||||
.take(MAX_OUTPUT_LINES_FOR_EXEC_TOOL_CALL)
|
||||
.collect::<Vec<_>>()
|
||||
@@ -363,6 +362,9 @@ impl EventProcessor for EventProcessorWithHumanOutput {
|
||||
}
|
||||
}
|
||||
}
|
||||
EventMsg::WebSearchBegin(WebSearchBeginEvent { call_id: _, query }) => {
|
||||
ts_println!(self, "🌐 {query}");
|
||||
}
|
||||
EventMsg::PatchApplyBegin(PatchApplyBeginEvent {
|
||||
call_id,
|
||||
auto_approved,
|
||||
|
||||
@@ -150,6 +150,7 @@ pub async fn run_main(cli: Cli, codex_linux_sandbox_exe: Option<PathBuf>) -> any
|
||||
include_apply_patch_tool: None,
|
||||
disable_response_storage: oss.then_some(true),
|
||||
show_raw_agent_reasoning: oss.then_some(true),
|
||||
tools_web_search_request: None,
|
||||
};
|
||||
// Parse `-c` overrides.
|
||||
let cli_kv_overrides = match config_overrides.parse_overrides() {
|
||||
|
||||
3
codex-rs/exec/tests/all.rs
Normal file
3
codex-rs/exec/tests/all.rs
Normal file
@@ -0,0 +1,3 @@
|
||||
// Single integration test binary that aggregates all test modules.
|
||||
// The submodules live in `tests/suite/`.
|
||||
mod suite;
|
||||
@@ -1,339 +0,0 @@
|
||||
#![allow(clippy::expect_used, clippy::unwrap_used)]
|
||||
|
||||
use anyhow::Context;
|
||||
use assert_cmd::prelude::*;
|
||||
use codex_core::CODEX_APPLY_PATCH_ARG1;
|
||||
use std::fs;
|
||||
use std::process::Command;
|
||||
use tempfile::tempdir;
|
||||
|
||||
/// While we may add an `apply-patch` subcommand to the `codex` CLI multitool
|
||||
/// at some point, we must ensure that the smaller `codex-exec` CLI can still
|
||||
/// emulate the `apply_patch` CLI.
|
||||
#[test]
|
||||
fn test_standalone_exec_cli_can_use_apply_patch() -> anyhow::Result<()> {
|
||||
let tmp = tempdir()?;
|
||||
let relative_path = "source.txt";
|
||||
let absolute_path = tmp.path().join(relative_path);
|
||||
fs::write(&absolute_path, "original content\n")?;
|
||||
|
||||
Command::cargo_bin("codex-exec")
|
||||
.context("should find binary for codex-exec")?
|
||||
.arg(CODEX_APPLY_PATCH_ARG1)
|
||||
.arg(
|
||||
r#"*** Begin Patch
|
||||
*** Update File: source.txt
|
||||
@@
|
||||
-original content
|
||||
+modified by apply_patch
|
||||
*** End Patch"#,
|
||||
)
|
||||
.current_dir(tmp.path())
|
||||
.assert()
|
||||
.success()
|
||||
.stdout("Success. Updated the following files:\nM source.txt\n")
|
||||
.stderr(predicates::str::is_empty());
|
||||
assert_eq!(
|
||||
fs::read_to_string(absolute_path)?,
|
||||
"modified by apply_patch\n"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(not(target_os = "windows"))]
|
||||
#[tokio::test]
|
||||
async fn test_apply_patch_tool() -> anyhow::Result<()> {
|
||||
use core_test_support::load_sse_fixture_with_id_from_str;
|
||||
use tempfile::TempDir;
|
||||
use wiremock::Mock;
|
||||
use wiremock::MockServer;
|
||||
use wiremock::ResponseTemplate;
|
||||
use wiremock::matchers::method;
|
||||
use wiremock::matchers::path;
|
||||
|
||||
const SSE_TOOL_CALL_ADD: &str = r#"[
|
||||
{
|
||||
"type": "response.output_item.done",
|
||||
"item": {
|
||||
"type": "function_call",
|
||||
"name": "apply_patch",
|
||||
"arguments": "{\n \"input\": \"*** Begin Patch\\n*** Add File: test.md\\n+Hello world\\n*** End Patch\"\n}",
|
||||
"call_id": "__ID__"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "response.completed",
|
||||
"response": {
|
||||
"id": "__ID__",
|
||||
"usage": {
|
||||
"input_tokens": 0,
|
||||
"input_tokens_details": null,
|
||||
"output_tokens": 0,
|
||||
"output_tokens_details": null,
|
||||
"total_tokens": 0
|
||||
},
|
||||
"output": []
|
||||
}
|
||||
}
|
||||
]"#;
|
||||
|
||||
const SSE_TOOL_CALL_UPDATE: &str = r#"[
|
||||
{
|
||||
"type": "response.output_item.done",
|
||||
"item": {
|
||||
"type": "function_call",
|
||||
"name": "apply_patch",
|
||||
"arguments": "{\n \"input\": \"*** Begin Patch\\n*** Update File: test.md\\n@@\\n-Hello world\\n+Final text\\n*** End Patch\"\n}",
|
||||
"call_id": "__ID__"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "response.completed",
|
||||
"response": {
|
||||
"id": "__ID__",
|
||||
"usage": {
|
||||
"input_tokens": 0,
|
||||
"input_tokens_details": null,
|
||||
"output_tokens": 0,
|
||||
"output_tokens_details": null,
|
||||
"total_tokens": 0
|
||||
},
|
||||
"output": []
|
||||
}
|
||||
}
|
||||
]"#;
|
||||
|
||||
const SSE_TOOL_CALL_COMPLETED: &str = r#"[
|
||||
{
|
||||
"type": "response.completed",
|
||||
"response": {
|
||||
"id": "__ID__",
|
||||
"usage": {
|
||||
"input_tokens": 0,
|
||||
"input_tokens_details": null,
|
||||
"output_tokens": 0,
|
||||
"output_tokens_details": null,
|
||||
"total_tokens": 0
|
||||
},
|
||||
"output": []
|
||||
}
|
||||
}
|
||||
]"#;
|
||||
|
||||
// Start a mock model server
|
||||
let server = MockServer::start().await;
|
||||
|
||||
// First response: model calls apply_patch to create test.md
|
||||
let first = ResponseTemplate::new(200)
|
||||
.insert_header("content-type", "text/event-stream")
|
||||
.set_body_raw(
|
||||
load_sse_fixture_with_id_from_str(SSE_TOOL_CALL_ADD, "call1"),
|
||||
"text/event-stream",
|
||||
);
|
||||
|
||||
Mock::given(method("POST"))
|
||||
// .and(path("/v1/responses"))
|
||||
.respond_with(first)
|
||||
.up_to_n_times(1)
|
||||
.mount(&server)
|
||||
.await;
|
||||
|
||||
// Second response: model calls apply_patch to update test.md
|
||||
let second = ResponseTemplate::new(200)
|
||||
.insert_header("content-type", "text/event-stream")
|
||||
.set_body_raw(
|
||||
load_sse_fixture_with_id_from_str(SSE_TOOL_CALL_UPDATE, "call2"),
|
||||
"text/event-stream",
|
||||
);
|
||||
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/v1/responses"))
|
||||
.respond_with(second)
|
||||
.up_to_n_times(1)
|
||||
.mount(&server)
|
||||
.await;
|
||||
|
||||
let final_completed = ResponseTemplate::new(200)
|
||||
.insert_header("content-type", "text/event-stream")
|
||||
.set_body_raw(
|
||||
load_sse_fixture_with_id_from_str(SSE_TOOL_CALL_COMPLETED, "resp3"),
|
||||
"text/event-stream",
|
||||
);
|
||||
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/v1/responses"))
|
||||
.respond_with(final_completed)
|
||||
.expect(1)
|
||||
.mount(&server)
|
||||
.await;
|
||||
|
||||
let tmp_cwd = TempDir::new().unwrap();
|
||||
Command::cargo_bin("codex-exec")
|
||||
.context("should find binary for codex-exec")?
|
||||
.current_dir(tmp_cwd.path())
|
||||
.env("CODEX_HOME", tmp_cwd.path())
|
||||
.env("OPENAI_API_KEY", "dummy")
|
||||
.env("OPENAI_BASE_URL", format!("{}/v1", server.uri()))
|
||||
.arg("--skip-git-repo-check")
|
||||
.arg("-s")
|
||||
.arg("workspace-write")
|
||||
.arg("foo")
|
||||
.assert()
|
||||
.success();
|
||||
|
||||
// Verify final file contents
|
||||
let final_path = tmp_cwd.path().join("test.md");
|
||||
let contents = std::fs::read_to_string(&final_path)
|
||||
.unwrap_or_else(|e| panic!("failed reading {}: {e}", final_path.display()));
|
||||
assert_eq!(contents, "Final text\n");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(not(target_os = "windows"))]
|
||||
#[tokio::test]
|
||||
async fn test_apply_patch_freeform_tool() -> anyhow::Result<()> {
|
||||
use core_test_support::load_sse_fixture_with_id_from_str;
|
||||
use tempfile::TempDir;
|
||||
use wiremock::Mock;
|
||||
use wiremock::MockServer;
|
||||
use wiremock::ResponseTemplate;
|
||||
use wiremock::matchers::method;
|
||||
use wiremock::matchers::path;
|
||||
|
||||
const SSE_TOOL_CALL_ADD: &str = r#"[
|
||||
{
|
||||
"type": "response.output_item.done",
|
||||
"item": {
|
||||
"type": "custom_tool_call",
|
||||
"name": "apply_patch",
|
||||
"input": "*** Begin Patch\n*** Add File: test.md\n+Hello world\n*** End Patch",
|
||||
"call_id": "__ID__"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "response.completed",
|
||||
"response": {
|
||||
"id": "__ID__",
|
||||
"usage": {
|
||||
"input_tokens": 0,
|
||||
"input_tokens_details": null,
|
||||
"output_tokens": 0,
|
||||
"output_tokens_details": null,
|
||||
"total_tokens": 0
|
||||
},
|
||||
"output": []
|
||||
}
|
||||
}
|
||||
]"#;
|
||||
|
||||
const SSE_TOOL_CALL_UPDATE: &str = r#"[
|
||||
{
|
||||
"type": "response.output_item.done",
|
||||
"item": {
|
||||
"type": "custom_tool_call",
|
||||
"name": "apply_patch",
|
||||
"input": "*** Begin Patch\n*** Update File: test.md\n@@\n-Hello world\n+Final text\n*** End Patch",
|
||||
"call_id": "__ID__"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "response.completed",
|
||||
"response": {
|
||||
"id": "__ID__",
|
||||
"usage": {
|
||||
"input_tokens": 0,
|
||||
"input_tokens_details": null,
|
||||
"output_tokens": 0,
|
||||
"output_tokens_details": null,
|
||||
"total_tokens": 0
|
||||
},
|
||||
"output": []
|
||||
}
|
||||
}
|
||||
]"#;
|
||||
|
||||
const SSE_TOOL_CALL_COMPLETED: &str = r#"[
|
||||
{
|
||||
"type": "response.completed",
|
||||
"response": {
|
||||
"id": "__ID__",
|
||||
"usage": {
|
||||
"input_tokens": 0,
|
||||
"input_tokens_details": null,
|
||||
"output_tokens": 0,
|
||||
"output_tokens_details": null,
|
||||
"total_tokens": 0
|
||||
},
|
||||
"output": []
|
||||
}
|
||||
}
|
||||
]"#;
|
||||
|
||||
// Start a mock model server
|
||||
let server = MockServer::start().await;
|
||||
|
||||
// First response: model calls apply_patch to create test.md
|
||||
let first = ResponseTemplate::new(200)
|
||||
.insert_header("content-type", "text/event-stream")
|
||||
.set_body_raw(
|
||||
load_sse_fixture_with_id_from_str(SSE_TOOL_CALL_ADD, "call1"),
|
||||
"text/event-stream",
|
||||
);
|
||||
|
||||
Mock::given(method("POST"))
|
||||
// .and(path("/v1/responses"))
|
||||
.respond_with(first)
|
||||
.up_to_n_times(1)
|
||||
.mount(&server)
|
||||
.await;
|
||||
|
||||
// Second response: model calls apply_patch to update test.md
|
||||
let second = ResponseTemplate::new(200)
|
||||
.insert_header("content-type", "text/event-stream")
|
||||
.set_body_raw(
|
||||
load_sse_fixture_with_id_from_str(SSE_TOOL_CALL_UPDATE, "call2"),
|
||||
"text/event-stream",
|
||||
);
|
||||
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/v1/responses"))
|
||||
.respond_with(second)
|
||||
.up_to_n_times(1)
|
||||
.mount(&server)
|
||||
.await;
|
||||
|
||||
let final_completed = ResponseTemplate::new(200)
|
||||
.insert_header("content-type", "text/event-stream")
|
||||
.set_body_raw(
|
||||
load_sse_fixture_with_id_from_str(SSE_TOOL_CALL_COMPLETED, "resp3"),
|
||||
"text/event-stream",
|
||||
);
|
||||
|
||||
Mock::given(method("POST"))
|
||||
// .and(path("/v1/responses"))
|
||||
.respond_with(final_completed)
|
||||
.expect(1)
|
||||
.mount(&server)
|
||||
.await;
|
||||
|
||||
let tmp_cwd = TempDir::new().unwrap();
|
||||
Command::cargo_bin("codex-exec")
|
||||
.context("should find binary for codex-exec")?
|
||||
.current_dir(tmp_cwd.path())
|
||||
.env("CODEX_HOME", tmp_cwd.path())
|
||||
.env("OPENAI_API_KEY", "dummy")
|
||||
.env("OPENAI_BASE_URL", format!("{}/v1", server.uri()))
|
||||
.arg("--skip-git-repo-check")
|
||||
.arg("-s")
|
||||
.arg("workspace-write")
|
||||
.arg("foo")
|
||||
.assert()
|
||||
.success();
|
||||
|
||||
// Verify final file contents
|
||||
let final_path = tmp_cwd.path().join("test.md");
|
||||
let contents = std::fs::read_to_string(&final_path)
|
||||
.unwrap_or_else(|e| panic!("failed reading {}: {e}", final_path.display()));
|
||||
assert_eq!(contents, "Final text\n");
|
||||
Ok(())
|
||||
}
|
||||
4
codex-rs/exec/tests/fixtures/apply_patch_freeform_final.txt
vendored
Normal file
4
codex-rs/exec/tests/fixtures/apply_patch_freeform_final.txt
vendored
Normal file
@@ -0,0 +1,4 @@
|
||||
class BaseClass:
|
||||
def method():
|
||||
|
||||
return True
|
||||
25
codex-rs/exec/tests/fixtures/sse_apply_patch_add.json
vendored
Normal file
25
codex-rs/exec/tests/fixtures/sse_apply_patch_add.json
vendored
Normal file
@@ -0,0 +1,25 @@
|
||||
[
|
||||
{
|
||||
"type": "response.output_item.done",
|
||||
"item": {
|
||||
"type": "custom_tool_call",
|
||||
"name": "apply_patch",
|
||||
"input": "*** Begin Patch\n*** Add File: test.md\n+Hello world\n*** End Patch",
|
||||
"call_id": "__ID__"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "response.completed",
|
||||
"response": {
|
||||
"id": "__ID__",
|
||||
"usage": {
|
||||
"input_tokens": 0,
|
||||
"input_tokens_details": null,
|
||||
"output_tokens": 0,
|
||||
"output_tokens_details": null,
|
||||
"total_tokens": 0
|
||||
},
|
||||
"output": []
|
||||
}
|
||||
}
|
||||
]
|
||||
25
codex-rs/exec/tests/fixtures/sse_apply_patch_freeform_add.json
vendored
Normal file
25
codex-rs/exec/tests/fixtures/sse_apply_patch_freeform_add.json
vendored
Normal file
@@ -0,0 +1,25 @@
|
||||
[
|
||||
{
|
||||
"type": "response.output_item.done",
|
||||
"item": {
|
||||
"type": "custom_tool_call",
|
||||
"name": "apply_patch",
|
||||
"input": "*** Begin Patch\n*** Add File: app.py\n+class BaseClass:\n+ def method():\n+ return False\n*** End Patch",
|
||||
"call_id": "__ID__"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "response.completed",
|
||||
"response": {
|
||||
"id": "__ID__",
|
||||
"usage": {
|
||||
"input_tokens": 0,
|
||||
"input_tokens_details": null,
|
||||
"output_tokens": 0,
|
||||
"output_tokens_details": null,
|
||||
"total_tokens": 0
|
||||
},
|
||||
"output": []
|
||||
}
|
||||
}
|
||||
]
|
||||
25
codex-rs/exec/tests/fixtures/sse_apply_patch_freeform_update.json
vendored
Normal file
25
codex-rs/exec/tests/fixtures/sse_apply_patch_freeform_update.json
vendored
Normal file
@@ -0,0 +1,25 @@
|
||||
[
|
||||
{
|
||||
"type": "response.output_item.done",
|
||||
"item": {
|
||||
"type": "custom_tool_call",
|
||||
"name": "apply_patch",
|
||||
"input": "*** Begin Patch\n*** Update File: app.py\n@@ def method():\n- return False\n+\n+ return True\n*** End Patch",
|
||||
"call_id": "__ID__"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "response.completed",
|
||||
"response": {
|
||||
"id": "__ID__",
|
||||
"usage": {
|
||||
"input_tokens": 0,
|
||||
"input_tokens_details": null,
|
||||
"output_tokens": 0,
|
||||
"output_tokens_details": null,
|
||||
"total_tokens": 0
|
||||
},
|
||||
"output": []
|
||||
}
|
||||
}
|
||||
]
|
||||
25
codex-rs/exec/tests/fixtures/sse_apply_patch_update.json
vendored
Normal file
25
codex-rs/exec/tests/fixtures/sse_apply_patch_update.json
vendored
Normal file
@@ -0,0 +1,25 @@
|
||||
[
|
||||
{
|
||||
"type": "response.output_item.done",
|
||||
"item": {
|
||||
"type": "function_call",
|
||||
"name": "apply_patch",
|
||||
"arguments": "{\n \"input\": \"*** Begin Patch\\n*** Update File: test.md\\n@@\\n-Hello world\\n+Final text\\n*** End Patch\"\n}",
|
||||
"call_id": "__ID__"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "response.completed",
|
||||
"response": {
|
||||
"id": "__ID__",
|
||||
"usage": {
|
||||
"input_tokens": 0,
|
||||
"input_tokens_details": null,
|
||||
"output_tokens": 0,
|
||||
"output_tokens_details": null,
|
||||
"total_tokens": 0
|
||||
},
|
||||
"output": []
|
||||
}
|
||||
}
|
||||
]
|
||||
16
codex-rs/exec/tests/fixtures/sse_response_completed.json
vendored
Normal file
16
codex-rs/exec/tests/fixtures/sse_response_completed.json
vendored
Normal file
@@ -0,0 +1,16 @@
|
||||
[
|
||||
{
|
||||
"type": "response.completed",
|
||||
"response": {
|
||||
"id": "__ID__",
|
||||
"usage": {
|
||||
"input_tokens": 0,
|
||||
"input_tokens_details": null,
|
||||
"output_tokens": 0,
|
||||
"output_tokens_details": null,
|
||||
"total_tokens": 0
|
||||
},
|
||||
"output": []
|
||||
}
|
||||
}
|
||||
]
|
||||
108
codex-rs/exec/tests/suite/apply_patch.rs
Normal file
108
codex-rs/exec/tests/suite/apply_patch.rs
Normal file
@@ -0,0 +1,108 @@
|
||||
#![allow(clippy::expect_used, clippy::unwrap_used)]
|
||||
|
||||
use anyhow::Context;
|
||||
use assert_cmd::prelude::*;
|
||||
use codex_core::CODEX_APPLY_PATCH_ARG1;
|
||||
use std::fs;
|
||||
use std::process::Command;
|
||||
use tempfile::tempdir;
|
||||
|
||||
/// While we may add an `apply-patch` subcommand to the `codex` CLI multitool
|
||||
/// at some point, we must ensure that the smaller `codex-exec` CLI can still
|
||||
/// emulate the `apply_patch` CLI.
|
||||
#[test]
|
||||
fn test_standalone_exec_cli_can_use_apply_patch() -> anyhow::Result<()> {
|
||||
let tmp = tempdir()?;
|
||||
let relative_path = "source.txt";
|
||||
let absolute_path = tmp.path().join(relative_path);
|
||||
fs::write(&absolute_path, "original content\n")?;
|
||||
|
||||
Command::cargo_bin("codex-exec")
|
||||
.context("should find binary for codex-exec")?
|
||||
.arg(CODEX_APPLY_PATCH_ARG1)
|
||||
.arg(
|
||||
r#"*** Begin Patch
|
||||
*** Update File: source.txt
|
||||
@@
|
||||
-original content
|
||||
+modified by apply_patch
|
||||
*** End Patch"#,
|
||||
)
|
||||
.current_dir(tmp.path())
|
||||
.assert()
|
||||
.success()
|
||||
.stdout("Success. Updated the following files:\nM source.txt\n")
|
||||
.stderr(predicates::str::is_empty());
|
||||
assert_eq!(
|
||||
fs::read_to_string(absolute_path)?,
|
||||
"modified by apply_patch\n"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(not(target_os = "windows"))]
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn test_apply_patch_tool() -> anyhow::Result<()> {
|
||||
use crate::suite::common::run_e2e_exec_test;
|
||||
use codex_core::spawn::CODEX_SANDBOX_NETWORK_DISABLED_ENV_VAR;
|
||||
|
||||
if std::env::var(CODEX_SANDBOX_NETWORK_DISABLED_ENV_VAR).is_ok() {
|
||||
println!(
|
||||
"Skipping test because it cannot execute when network is disabled in a Codex sandbox."
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let tmp_cwd = tempdir().expect("failed to create temp dir");
|
||||
let tmp_path = tmp_cwd.path().to_path_buf();
|
||||
run_e2e_exec_test(
|
||||
tmp_cwd.path(),
|
||||
vec![
|
||||
include_str!("../fixtures/sse_apply_patch_add.json").to_string(),
|
||||
include_str!("../fixtures/sse_apply_patch_update.json").to_string(),
|
||||
include_str!("../fixtures/sse_response_completed.json").to_string(),
|
||||
],
|
||||
)
|
||||
.await;
|
||||
|
||||
let final_path = tmp_path.join("test.md");
|
||||
let contents = std::fs::read_to_string(&final_path)
|
||||
.unwrap_or_else(|e| panic!("failed reading {}: {e}", final_path.display()));
|
||||
assert_eq!(contents, "Final text\n");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(not(target_os = "windows"))]
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn test_apply_patch_freeform_tool() -> anyhow::Result<()> {
|
||||
use crate::suite::common::run_e2e_exec_test;
|
||||
use codex_core::spawn::CODEX_SANDBOX_NETWORK_DISABLED_ENV_VAR;
|
||||
|
||||
if std::env::var(CODEX_SANDBOX_NETWORK_DISABLED_ENV_VAR).is_ok() {
|
||||
println!(
|
||||
"Skipping test because it cannot execute when network is disabled in a Codex sandbox."
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let tmp_cwd = tempdir().expect("failed to create temp dir");
|
||||
run_e2e_exec_test(
|
||||
tmp_cwd.path(),
|
||||
vec![
|
||||
include_str!("../fixtures/sse_apply_patch_freeform_add.json").to_string(),
|
||||
include_str!("../fixtures/sse_apply_patch_freeform_update.json").to_string(),
|
||||
include_str!("../fixtures/sse_response_completed.json").to_string(),
|
||||
],
|
||||
)
|
||||
.await;
|
||||
|
||||
// Verify final file contents
|
||||
let final_path = tmp_cwd.path().join("app.py");
|
||||
let contents = std::fs::read_to_string(&final_path)
|
||||
.unwrap_or_else(|e| panic!("failed reading {}: {e}", final_path.display()));
|
||||
assert_eq!(
|
||||
contents,
|
||||
include_str!("../fixtures/apply_patch_freeform_final.txt")
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
73
codex-rs/exec/tests/suite/common.rs
Normal file
73
codex-rs/exec/tests/suite/common.rs
Normal file
@@ -0,0 +1,73 @@
|
||||
// this file is only used for e2e tests which are currently disabled on windows
|
||||
#![cfg(not(target_os = "windows"))]
|
||||
#![allow(clippy::expect_used)]
|
||||
|
||||
use anyhow::Context;
|
||||
use assert_cmd::prelude::*;
|
||||
use core_test_support::load_sse_fixture_with_id_from_str;
|
||||
use std::path::Path;
|
||||
use std::process::Command;
|
||||
use std::sync::atomic::AtomicUsize;
|
||||
use std::sync::atomic::Ordering;
|
||||
use wiremock::Mock;
|
||||
use wiremock::MockServer;
|
||||
use wiremock::matchers::method;
|
||||
use wiremock::matchers::path;
|
||||
|
||||
use wiremock::Respond;
|
||||
|
||||
struct SeqResponder {
|
||||
num_calls: AtomicUsize,
|
||||
responses: Vec<String>,
|
||||
}
|
||||
|
||||
impl Respond for SeqResponder {
|
||||
fn respond(&self, _: &wiremock::Request) -> wiremock::ResponseTemplate {
|
||||
let call_num = self.num_calls.fetch_add(1, Ordering::SeqCst);
|
||||
match self.responses.get(call_num) {
|
||||
Some(body) => wiremock::ResponseTemplate::new(200)
|
||||
.insert_header("content-type", "text/event-stream")
|
||||
.set_body_raw(
|
||||
load_sse_fixture_with_id_from_str(body, &format!("request_{}", call_num)),
|
||||
"text/event-stream",
|
||||
),
|
||||
None => panic!("no response for {call_num}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Helper function to run an E2E test of a codex-exec call. Starts a wiremock
|
||||
/// server, and returns the response_streams in order for each api call. Runs
|
||||
/// the codex-exec command with the wiremock server as the model server.
|
||||
pub(crate) async fn run_e2e_exec_test(cwd: &Path, response_streams: Vec<String>) {
|
||||
let server = MockServer::start().await;
|
||||
|
||||
let num_calls = response_streams.len();
|
||||
let seq_responder = SeqResponder {
|
||||
num_calls: AtomicUsize::new(0),
|
||||
responses: response_streams,
|
||||
};
|
||||
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/v1/responses"))
|
||||
.respond_with(seq_responder)
|
||||
.expect(num_calls as u64)
|
||||
.mount(&server)
|
||||
.await;
|
||||
|
||||
let cwd = cwd.to_path_buf();
|
||||
let uri = server.uri();
|
||||
Command::cargo_bin("codex-exec")
|
||||
.context("should find binary for codex-exec")
|
||||
.expect("should find binary for codex-exec")
|
||||
.current_dir(cwd.clone())
|
||||
.env("CODEX_HOME", cwd.clone())
|
||||
.env("OPENAI_API_KEY", "dummy")
|
||||
.env("OPENAI_BASE_URL", format!("{}/v1", uri))
|
||||
.arg("--skip-git-repo-check")
|
||||
.arg("-s")
|
||||
.arg("danger-full-access")
|
||||
.arg("foo")
|
||||
.assert()
|
||||
.success();
|
||||
}
|
||||
4
codex-rs/exec/tests/suite/mod.rs
Normal file
4
codex-rs/exec/tests/suite/mod.rs
Normal file
@@ -0,0 +1,4 @@
|
||||
// Aggregates all former standalone integration tests as modules.
|
||||
mod apply_patch;
|
||||
mod common;
|
||||
mod sandbox;
|
||||
3
codex-rs/execpolicy/tests/all.rs
Normal file
3
codex-rs/execpolicy/tests/all.rs
Normal file
@@ -0,0 +1,3 @@
|
||||
// Single integration test binary that aggregates all test modules.
|
||||
// The submodules live in `tests/suite/`.
|
||||
mod suite;
|
||||
10
codex-rs/execpolicy/tests/suite/mod.rs
Normal file
10
codex-rs/execpolicy/tests/suite/mod.rs
Normal file
@@ -0,0 +1,10 @@
|
||||
// Aggregates all former standalone integration tests as modules.
|
||||
mod bad;
|
||||
mod cp;
|
||||
mod good;
|
||||
mod head;
|
||||
mod literal;
|
||||
mod ls;
|
||||
mod parse_sed_command;
|
||||
mod pwd;
|
||||
mod sed;
|
||||
3
codex-rs/linux-sandbox/tests/all.rs
Normal file
3
codex-rs/linux-sandbox/tests/all.rs
Normal file
@@ -0,0 +1,3 @@
|
||||
// Single integration test binary that aggregates all test modules.
|
||||
// The submodules live in `tests/suite/`.
|
||||
mod suite;
|
||||
2
codex-rs/linux-sandbox/tests/suite/mod.rs
Normal file
2
codex-rs/linux-sandbox/tests/suite/mod.rs
Normal file
@@ -0,0 +1,2 @@
|
||||
// Aggregates all former standalone integration tests as modules.
|
||||
mod landlock;
|
||||
3
codex-rs/login/tests/all.rs
Normal file
3
codex-rs/login/tests/all.rs
Normal file
@@ -0,0 +1,3 @@
|
||||
// Single integration test binary that aggregates all test modules.
|
||||
// The submodules live in `tests/suite/`.
|
||||
mod suite;
|
||||
2
codex-rs/login/tests/suite/mod.rs
Normal file
2
codex-rs/login/tests/suite/mod.rs
Normal file
@@ -0,0 +1,2 @@
|
||||
// Aggregates all former standalone integration tests as modules.
|
||||
mod login_server_e2e;
|
||||
@@ -738,6 +738,7 @@ fn derive_config_from_params(
|
||||
include_apply_patch_tool,
|
||||
disable_response_storage: None,
|
||||
show_raw_agent_reasoning: None,
|
||||
tools_web_search_request: None,
|
||||
};
|
||||
|
||||
let cli_overrides = cli_overrides
|
||||
|
||||
@@ -163,6 +163,7 @@ impl CodexToolCallParam {
|
||||
include_apply_patch_tool: None,
|
||||
disable_response_storage: None,
|
||||
show_raw_agent_reasoning: None,
|
||||
tools_web_search_request: None,
|
||||
};
|
||||
|
||||
let cli_overrides = cli_overrides
|
||||
|
||||
@@ -272,6 +272,7 @@ async fn run_codex_tool_session_inner(
|
||||
| EventMsg::PatchApplyBegin(_)
|
||||
| EventMsg::PatchApplyEnd(_)
|
||||
| EventMsg::TurnDiff(_)
|
||||
| EventMsg::WebSearchBegin(_)
|
||||
| EventMsg::GetHistoryEntryResponse(_)
|
||||
| EventMsg::PlanUpdate(_)
|
||||
| EventMsg::TurnAborted(_)
|
||||
|
||||
3
codex-rs/mcp-server/tests/all.rs
Normal file
3
codex-rs/mcp-server/tests/all.rs
Normal file
@@ -0,0 +1,3 @@
|
||||
// Single integration test binary that aggregates all test modules.
|
||||
// The submodules live in `tests/suite/`.
|
||||
mod suite;
|
||||
8
codex-rs/mcp-server/tests/suite/mod.rs
Normal file
8
codex-rs/mcp-server/tests/suite/mod.rs
Normal file
@@ -0,0 +1,8 @@
|
||||
// Aggregates all former standalone integration tests as modules.
|
||||
mod auth;
|
||||
mod codex_message_processor_flow;
|
||||
mod codex_tool;
|
||||
mod create_conversation;
|
||||
mod interrupt;
|
||||
mod login;
|
||||
mod send_message;
|
||||
3
codex-rs/mcp-types/tests/all.rs
Normal file
3
codex-rs/mcp-types/tests/all.rs
Normal file
@@ -0,0 +1,3 @@
|
||||
// Single integration test binary that aggregates all test modules.
|
||||
// The submodules live in `tests/suite/`.
|
||||
mod suite;
|
||||
3
codex-rs/mcp-types/tests/suite/mod.rs
Normal file
3
codex-rs/mcp-types/tests/suite/mod.rs
Normal file
@@ -0,0 +1,3 @@
|
||||
// Aggregates all former standalone integration tests as modules.
|
||||
mod initialize;
|
||||
mod progress_notification;
|
||||
@@ -48,6 +48,8 @@ pub fn generate_ts(out_dir: &Path, prettier: Option<&Path>) -> Result<()> {
|
||||
codex_protocol::mcp_protocol::ExecCommandApprovalResponse::export_all_to(out_dir)?;
|
||||
codex_protocol::mcp_protocol::ServerNotification::export_all_to(out_dir)?;
|
||||
|
||||
generate_index_ts(out_dir)?;
|
||||
|
||||
// Prepend header to each generated .ts file
|
||||
let ts_files = ts_files_in(out_dir)?;
|
||||
for file in &ts_files {
|
||||
@@ -109,5 +111,39 @@ fn ts_files_in(dir: &Path) -> Result<Vec<PathBuf>> {
|
||||
files.push(path);
|
||||
}
|
||||
}
|
||||
files.sort();
|
||||
Ok(files)
|
||||
}
|
||||
|
||||
/// Generate an index.ts file that re-exports all generated types.
|
||||
/// This allows consumers to import all types from a single file.
|
||||
fn generate_index_ts(out_dir: &Path) -> Result<PathBuf> {
|
||||
let mut entries: Vec<String> = Vec::new();
|
||||
let mut stems: Vec<String> = ts_files_in(out_dir)?
|
||||
.into_iter()
|
||||
.filter_map(|p| {
|
||||
let stem = p.file_stem()?.to_string_lossy().into_owned();
|
||||
if stem == "index" { None } else { Some(stem) }
|
||||
})
|
||||
.collect();
|
||||
stems.sort();
|
||||
stems.dedup();
|
||||
|
||||
for name in stems {
|
||||
entries.push(format!("export type {{ {name} }} from \"./{name}\";\n"));
|
||||
}
|
||||
|
||||
let mut content =
|
||||
String::with_capacity(HEADER.len() + entries.iter().map(|s| s.len()).sum::<usize>());
|
||||
content.push_str(HEADER);
|
||||
for line in &entries {
|
||||
content.push_str(line);
|
||||
}
|
||||
|
||||
let index_path = out_dir.join("index.ts");
|
||||
let mut f = fs::File::create(&index_path)
|
||||
.with_context(|| format!("Failed to create {}", index_path.display()))?;
|
||||
f.write_all(content.as_bytes())
|
||||
.with_context(|| format!("Failed to write {}", index_path.display()))?;
|
||||
Ok(index_path)
|
||||
}
|
||||
|
||||
@@ -437,6 +437,8 @@ pub enum EventMsg {
|
||||
|
||||
McpToolCallEnd(McpToolCallEndEvent),
|
||||
|
||||
WebSearchBegin(WebSearchBeginEvent),
|
||||
|
||||
/// Notification that the server is about to execute a command.
|
||||
ExecCommandBegin(ExecCommandBeginEvent),
|
||||
|
||||
@@ -658,6 +660,12 @@ impl McpToolCallEndEvent {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct WebSearchBeginEvent {
|
||||
pub call_id: String,
|
||||
pub query: String,
|
||||
}
|
||||
|
||||
/// Response payload for `Op::GetHistory` containing the current session's
|
||||
/// in-memory transcript.
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
@@ -685,6 +693,9 @@ pub struct ExecCommandEndEvent {
|
||||
pub stdout: String,
|
||||
/// Captured stderr
|
||||
pub stderr: String,
|
||||
/// Captured aggregated output
|
||||
#[serde(default)]
|
||||
pub aggregated_output: String,
|
||||
/// The command's exit code.
|
||||
pub exit_code: i32,
|
||||
/// The duration of the command execution.
|
||||
|
||||
@@ -40,7 +40,10 @@ codex-login = { path = "../login" }
|
||||
codex-ollama = { path = "../ollama" }
|
||||
codex-protocol = { path = "../protocol" }
|
||||
color-eyre = "0.6.3"
|
||||
crossterm = { version = "0.28.1", features = ["bracketed-paste", "event-stream"] }
|
||||
crossterm = { version = "0.28.1", features = [
|
||||
"bracketed-paste",
|
||||
"event-stream",
|
||||
] }
|
||||
diffy = "0.4.2"
|
||||
image = { version = "^0.25.6", default-features = false, features = [
|
||||
"jpeg",
|
||||
@@ -82,6 +85,7 @@ tui-input = "0.14.0"
|
||||
tui-markdown = "0.3.3"
|
||||
unicode-segmentation = "1.12.0"
|
||||
unicode-width = "0.1"
|
||||
url = "2"
|
||||
uuid = "1"
|
||||
|
||||
[target.'cfg(unix)'.dependencies]
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
use crate::app_backtrack::BacktrackState;
|
||||
use crate::app_event::AppEvent;
|
||||
use crate::app_event_sender::AppEventSender;
|
||||
use crate::chatwidget::ChatWidget;
|
||||
@@ -25,27 +26,31 @@ use std::thread;
|
||||
use std::time::Duration;
|
||||
use tokio::select;
|
||||
use tokio::sync::mpsc::unbounded_channel;
|
||||
// use uuid::Uuid;
|
||||
|
||||
pub(crate) struct App {
|
||||
server: Arc<ConversationManager>,
|
||||
app_event_tx: AppEventSender,
|
||||
chat_widget: ChatWidget,
|
||||
pub(crate) server: Arc<ConversationManager>,
|
||||
pub(crate) app_event_tx: AppEventSender,
|
||||
pub(crate) chat_widget: ChatWidget,
|
||||
|
||||
/// Config is stored here so we can recreate ChatWidgets as needed.
|
||||
config: Config,
|
||||
pub(crate) config: Config,
|
||||
|
||||
file_search: FileSearchManager,
|
||||
pub(crate) file_search: FileSearchManager,
|
||||
|
||||
transcript_lines: Vec<Line<'static>>,
|
||||
pub(crate) transcript_lines: Vec<Line<'static>>,
|
||||
|
||||
// Transcript overlay state
|
||||
transcript_overlay: Option<TranscriptApp>,
|
||||
deferred_history_lines: Vec<Line<'static>>,
|
||||
pub(crate) transcript_overlay: Option<TranscriptApp>,
|
||||
pub(crate) deferred_history_lines: Vec<Line<'static>>,
|
||||
|
||||
enhanced_keys_supported: bool,
|
||||
pub(crate) enhanced_keys_supported: bool,
|
||||
|
||||
/// Controls the animation thread that sends CommitTick events.
|
||||
commit_anim_running: Arc<AtomicBool>,
|
||||
pub(crate) commit_anim_running: Arc<AtomicBool>,
|
||||
|
||||
// Esc-backtracking state grouped
|
||||
pub(crate) backtrack: crate::app_backtrack::BacktrackState,
|
||||
}
|
||||
|
||||
impl App {
|
||||
@@ -87,6 +92,7 @@ impl App {
|
||||
transcript_overlay: None,
|
||||
deferred_history_lines: Vec::new(),
|
||||
commit_anim_running: Arc::new(AtomicBool::new(false)),
|
||||
backtrack: BacktrackState::default(),
|
||||
};
|
||||
|
||||
let tui_events = tui.event_stream();
|
||||
@@ -96,7 +102,7 @@ impl App {
|
||||
|
||||
while select! {
|
||||
Some(event) = app_event_rx.recv() => {
|
||||
app.handle_event(tui, event)?
|
||||
app.handle_event(tui, event).await?
|
||||
}
|
||||
Some(event) = tui_events.next() => {
|
||||
app.handle_tui_event(tui, event).await?
|
||||
@@ -111,18 +117,8 @@ impl App {
|
||||
tui: &mut tui::Tui,
|
||||
event: TuiEvent,
|
||||
) -> Result<bool> {
|
||||
if let Some(overlay) = &mut self.transcript_overlay {
|
||||
overlay.handle_event(tui, event)?;
|
||||
if overlay.is_done {
|
||||
// Exit alternate screen and restore viewport.
|
||||
let _ = tui.leave_alt_screen();
|
||||
if !self.deferred_history_lines.is_empty() {
|
||||
let lines = std::mem::take(&mut self.deferred_history_lines);
|
||||
tui.insert_history_lines(lines);
|
||||
}
|
||||
self.transcript_overlay = None;
|
||||
tui.frame_requester().schedule_frame();
|
||||
}
|
||||
if self.transcript_overlay.is_some() {
|
||||
let _ = self.handle_backtrack_overlay_event(tui, event).await?;
|
||||
} else {
|
||||
match event {
|
||||
TuiEvent::Key(key_event) => {
|
||||
@@ -161,7 +157,7 @@ impl App {
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
fn handle_event(&mut self, tui: &mut tui::Tui, event: AppEvent) -> Result<bool> {
|
||||
async fn handle_event(&mut self, tui: &mut tui::Tui, event: AppEvent) -> Result<bool> {
|
||||
match event {
|
||||
AppEvent::NewSession => {
|
||||
self.chat_widget = ChatWidget::new(
|
||||
@@ -227,6 +223,9 @@ impl App {
|
||||
AppEvent::CodexEvent(event) => {
|
||||
self.chat_widget.handle_codex_event(event);
|
||||
}
|
||||
AppEvent::ConversationHistory(ev) => {
|
||||
self.on_conversation_history_for_backtrack(tui, ev).await?;
|
||||
}
|
||||
AppEvent::ExitRequest => {
|
||||
return Ok(false);
|
||||
}
|
||||
@@ -304,10 +303,44 @@ impl App {
|
||||
self.transcript_overlay = Some(TranscriptApp::new(self.transcript_lines.clone()));
|
||||
tui.frame_requester().schedule_frame();
|
||||
}
|
||||
// Esc primes/advances backtracking only in normal (not working) mode
|
||||
// with an empty composer. In any other state, forward Esc so the
|
||||
// active UI (e.g. status indicator, modals, popups) handles it.
|
||||
KeyEvent {
|
||||
code: KeyCode::Esc,
|
||||
kind: KeyEventKind::Press | KeyEventKind::Repeat,
|
||||
..
|
||||
} => {
|
||||
if self.chat_widget.is_normal_backtrack_mode()
|
||||
&& self.chat_widget.composer_is_empty()
|
||||
{
|
||||
self.handle_backtrack_esc_key(tui);
|
||||
} else {
|
||||
self.chat_widget.handle_key_event(key_event);
|
||||
}
|
||||
}
|
||||
// Enter confirms backtrack when primed + count > 0. Otherwise pass to widget.
|
||||
KeyEvent {
|
||||
code: KeyCode::Enter,
|
||||
kind: KeyEventKind::Press,
|
||||
..
|
||||
} if self.backtrack.primed
|
||||
&& self.backtrack.count > 0
|
||||
&& self.chat_widget.composer_is_empty() =>
|
||||
{
|
||||
// Delegate to helper for clarity; preserves behavior.
|
||||
self.confirm_backtrack_from_main();
|
||||
}
|
||||
KeyEvent {
|
||||
kind: KeyEventKind::Press | KeyEventKind::Repeat,
|
||||
..
|
||||
} => {
|
||||
// Any non-Esc key press should cancel a primed backtrack.
|
||||
// This avoids stale "Esc-primed" state after the user starts typing
|
||||
// (even if they later backspace to empty).
|
||||
if key_event.code != KeyCode::Esc && self.backtrack.primed {
|
||||
self.reset_backtrack_state();
|
||||
}
|
||||
self.chat_widget.handle_key_event(key_event);
|
||||
}
|
||||
_ => {
|
||||
|
||||
349
codex-rs/tui/src/app_backtrack.rs
Normal file
349
codex-rs/tui/src/app_backtrack.rs
Normal file
@@ -0,0 +1,349 @@
|
||||
use crate::app::App;
|
||||
use crate::backtrack_helpers;
|
||||
use crate::transcript_app::TranscriptApp;
|
||||
use crate::tui;
|
||||
use crate::tui::TuiEvent;
|
||||
use codex_core::protocol::ConversationHistoryResponseEvent;
|
||||
use color_eyre::eyre::Result;
|
||||
use crossterm::event::KeyCode;
|
||||
use crossterm::event::KeyEvent;
|
||||
use crossterm::event::KeyEventKind;
|
||||
/// Aggregates all backtrack-related state used by the App.
|
||||
#[derive(Default)]
|
||||
pub(crate) struct BacktrackState {
|
||||
/// True when Esc has primed backtrack mode in the main view.
|
||||
pub(crate) primed: bool,
|
||||
/// Session id of the base conversation to fork from.
|
||||
pub(crate) base_id: Option<uuid::Uuid>,
|
||||
/// Current step count (Nth last user message).
|
||||
pub(crate) count: usize,
|
||||
/// True when the transcript overlay is showing a backtrack preview.
|
||||
pub(crate) overlay_preview_active: bool,
|
||||
/// Pending fork request: (base_id, drop_count, prefill).
|
||||
pub(crate) pending: Option<(uuid::Uuid, usize, String)>,
|
||||
}
|
||||
|
||||
impl App {
|
||||
/// Route overlay events when transcript overlay is active.
|
||||
/// - If backtrack preview is active: Esc steps selection; Enter confirms.
|
||||
/// - Otherwise: Esc begins preview; all other events forward to overlay.
|
||||
/// interactions (Esc to step target, Enter to confirm) and overlay lifecycle.
|
||||
pub(crate) async fn handle_backtrack_overlay_event(
|
||||
&mut self,
|
||||
tui: &mut tui::Tui,
|
||||
event: TuiEvent,
|
||||
) -> Result<bool> {
|
||||
if self.backtrack.overlay_preview_active {
|
||||
match event {
|
||||
TuiEvent::Key(KeyEvent {
|
||||
code: KeyCode::Esc,
|
||||
kind: KeyEventKind::Press | KeyEventKind::Repeat,
|
||||
..
|
||||
}) => {
|
||||
self.overlay_step_backtrack(tui, event)?;
|
||||
Ok(true)
|
||||
}
|
||||
TuiEvent::Key(KeyEvent {
|
||||
code: KeyCode::Enter,
|
||||
kind: KeyEventKind::Press,
|
||||
..
|
||||
}) => {
|
||||
self.overlay_confirm_backtrack(tui);
|
||||
Ok(true)
|
||||
}
|
||||
// Catchall: forward any other events to the overlay widget.
|
||||
_ => {
|
||||
self.overlay_forward_event(tui, event)?;
|
||||
Ok(true)
|
||||
}
|
||||
}
|
||||
} else if let TuiEvent::Key(KeyEvent {
|
||||
code: KeyCode::Esc,
|
||||
kind: KeyEventKind::Press | KeyEventKind::Repeat,
|
||||
..
|
||||
}) = event
|
||||
{
|
||||
// First Esc in transcript overlay: begin backtrack preview at latest user message.
|
||||
self.begin_overlay_backtrack_preview(tui);
|
||||
Ok(true)
|
||||
} else {
|
||||
// Not in backtrack mode: forward events to the overlay widget.
|
||||
self.overlay_forward_event(tui, event)?;
|
||||
Ok(true)
|
||||
}
|
||||
}
|
||||
|
||||
/// Handle global Esc presses for backtracking when no overlay is present.
|
||||
pub(crate) fn handle_backtrack_esc_key(&mut self, tui: &mut tui::Tui) {
|
||||
// Only handle backtracking when composer is empty to avoid clobbering edits.
|
||||
if self.chat_widget.composer_is_empty() {
|
||||
if !self.backtrack.primed {
|
||||
self.prime_backtrack();
|
||||
} else if self.transcript_overlay.is_none() {
|
||||
self.open_backtrack_preview(tui);
|
||||
} else if self.backtrack.overlay_preview_active {
|
||||
self.step_backtrack_and_highlight(tui);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Stage a backtrack and request conversation history from the agent.
|
||||
pub(crate) fn request_backtrack(
|
||||
&mut self,
|
||||
prefill: String,
|
||||
base_id: uuid::Uuid,
|
||||
drop_last_messages: usize,
|
||||
) {
|
||||
self.backtrack.pending = Some((base_id, drop_last_messages, prefill));
|
||||
self.app_event_tx.send(crate::app_event::AppEvent::CodexOp(
|
||||
codex_core::protocol::Op::GetHistory,
|
||||
));
|
||||
}
|
||||
|
||||
/// Open transcript overlay (enters alternate screen and shows full transcript).
|
||||
pub(crate) fn open_transcript_overlay(&mut self, tui: &mut tui::Tui) {
|
||||
let _ = tui.enter_alt_screen();
|
||||
self.transcript_overlay = Some(TranscriptApp::new(self.transcript_lines.clone()));
|
||||
tui.frame_requester().schedule_frame();
|
||||
}
|
||||
|
||||
/// Close transcript overlay and restore normal UI.
|
||||
pub(crate) fn close_transcript_overlay(&mut self, tui: &mut tui::Tui) {
|
||||
let _ = tui.leave_alt_screen();
|
||||
let was_backtrack = self.backtrack.overlay_preview_active;
|
||||
if !self.deferred_history_lines.is_empty() {
|
||||
let lines = std::mem::take(&mut self.deferred_history_lines);
|
||||
tui.insert_history_lines(lines);
|
||||
}
|
||||
self.transcript_overlay = None;
|
||||
self.backtrack.overlay_preview_active = false;
|
||||
if was_backtrack {
|
||||
// Ensure backtrack state is fully reset when overlay closes (e.g. via 'q').
|
||||
self.reset_backtrack_state();
|
||||
}
|
||||
}
|
||||
|
||||
/// Re-render the full transcript into the terminal scrollback in one call.
|
||||
/// Useful when switching sessions to ensure prior history remains visible.
|
||||
pub(crate) fn render_transcript_once(&mut self, tui: &mut tui::Tui) {
|
||||
if !self.transcript_lines.is_empty() {
|
||||
tui.insert_history_lines(self.transcript_lines.clone());
|
||||
}
|
||||
}
|
||||
|
||||
/// Initialize backtrack state and show composer hint.
|
||||
fn prime_backtrack(&mut self) {
|
||||
self.backtrack.primed = true;
|
||||
self.backtrack.count = 0;
|
||||
self.backtrack.base_id = self.chat_widget.session_id();
|
||||
self.chat_widget.show_esc_backtrack_hint();
|
||||
}
|
||||
|
||||
/// Open overlay and begin backtrack preview flow (first step + highlight).
|
||||
fn open_backtrack_preview(&mut self, tui: &mut tui::Tui) {
|
||||
self.open_transcript_overlay(tui);
|
||||
self.backtrack.overlay_preview_active = true;
|
||||
// Composer is hidden by overlay; clear its hint.
|
||||
self.chat_widget.clear_esc_backtrack_hint();
|
||||
self.step_backtrack_and_highlight(tui);
|
||||
}
|
||||
|
||||
/// When overlay is already open, begin preview mode and select latest user message.
|
||||
fn begin_overlay_backtrack_preview(&mut self, tui: &mut tui::Tui) {
|
||||
self.backtrack.primed = true;
|
||||
self.backtrack.base_id = self.chat_widget.session_id();
|
||||
self.backtrack.overlay_preview_active = true;
|
||||
let sel = self.compute_backtrack_selection(tui, 1);
|
||||
self.apply_backtrack_selection(sel);
|
||||
tui.frame_requester().schedule_frame();
|
||||
}
|
||||
|
||||
/// Step selection to the next older user message and update overlay.
|
||||
fn step_backtrack_and_highlight(&mut self, tui: &mut tui::Tui) {
|
||||
let next = self.backtrack.count.saturating_add(1);
|
||||
let sel = self.compute_backtrack_selection(tui, next);
|
||||
self.apply_backtrack_selection(sel);
|
||||
tui.frame_requester().schedule_frame();
|
||||
}
|
||||
|
||||
/// Compute normalized target, scroll offset, and highlight for requested step.
|
||||
fn compute_backtrack_selection(
|
||||
&self,
|
||||
tui: &tui::Tui,
|
||||
requested_n: usize,
|
||||
) -> (usize, Option<usize>, Option<(usize, usize)>) {
|
||||
let nth = backtrack_helpers::normalize_backtrack_n(&self.transcript_lines, requested_n);
|
||||
let header_idx =
|
||||
backtrack_helpers::find_nth_last_user_header_index(&self.transcript_lines, nth);
|
||||
let offset = header_idx.map(|idx| {
|
||||
backtrack_helpers::wrapped_offset_before(
|
||||
&self.transcript_lines,
|
||||
idx,
|
||||
tui.terminal.viewport_area.width,
|
||||
)
|
||||
});
|
||||
let hl = backtrack_helpers::highlight_range_for_nth_last_user(&self.transcript_lines, nth);
|
||||
(nth, offset, hl)
|
||||
}
|
||||
|
||||
/// Apply a computed backtrack selection to the overlay and internal counter.
|
||||
fn apply_backtrack_selection(
|
||||
&mut self,
|
||||
selection: (usize, Option<usize>, Option<(usize, usize)>),
|
||||
) {
|
||||
let (nth, offset, hl) = selection;
|
||||
self.backtrack.count = nth;
|
||||
if let Some(overlay) = &mut self.transcript_overlay {
|
||||
if let Some(off) = offset {
|
||||
overlay.scroll_offset = off;
|
||||
}
|
||||
overlay.set_highlight_range(hl);
|
||||
}
|
||||
}
|
||||
|
||||
/// Forward any event to the overlay and close it if done.
|
||||
fn overlay_forward_event(&mut self, tui: &mut tui::Tui, event: TuiEvent) -> Result<()> {
|
||||
if let Some(overlay) = &mut self.transcript_overlay {
|
||||
overlay.handle_event(tui, event)?;
|
||||
if overlay.is_done {
|
||||
self.close_transcript_overlay(tui);
|
||||
tui.frame_requester().schedule_frame();
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Handle Enter in overlay backtrack preview: confirm selection and reset state.
|
||||
fn overlay_confirm_backtrack(&mut self, tui: &mut tui::Tui) {
|
||||
if let Some(base_id) = self.backtrack.base_id {
|
||||
let drop_last_messages = self.backtrack.count;
|
||||
let prefill =
|
||||
backtrack_helpers::nth_last_user_text(&self.transcript_lines, drop_last_messages)
|
||||
.unwrap_or_default();
|
||||
self.close_transcript_overlay(tui);
|
||||
self.request_backtrack(prefill, base_id, drop_last_messages);
|
||||
}
|
||||
self.reset_backtrack_state();
|
||||
}
|
||||
|
||||
/// Handle Esc in overlay backtrack preview: step selection if armed, else forward.
|
||||
fn overlay_step_backtrack(&mut self, tui: &mut tui::Tui, event: TuiEvent) -> Result<()> {
|
||||
if self.backtrack.base_id.is_some() {
|
||||
self.step_backtrack_and_highlight(tui);
|
||||
} else {
|
||||
self.overlay_forward_event(tui, event)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Confirm a primed backtrack from the main view (no overlay visible).
|
||||
/// Computes the prefill from the selected user message and requests history.
|
||||
pub(crate) fn confirm_backtrack_from_main(&mut self) {
|
||||
if let Some(base_id) = self.backtrack.base_id {
|
||||
let drop_last_messages = self.backtrack.count;
|
||||
let prefill =
|
||||
backtrack_helpers::nth_last_user_text(&self.transcript_lines, drop_last_messages)
|
||||
.unwrap_or_default();
|
||||
self.request_backtrack(prefill, base_id, drop_last_messages);
|
||||
}
|
||||
self.reset_backtrack_state();
|
||||
}
|
||||
|
||||
/// Clear all backtrack-related state and composer hints.
|
||||
pub(crate) fn reset_backtrack_state(&mut self) {
|
||||
self.backtrack.primed = false;
|
||||
self.backtrack.base_id = None;
|
||||
self.backtrack.count = 0;
|
||||
// In case a hint is somehow still visible (e.g., race with overlay open/close).
|
||||
self.chat_widget.clear_esc_backtrack_hint();
|
||||
}
|
||||
|
||||
/// Handle a ConversationHistory response while a backtrack is pending.
|
||||
/// If it matches the primed base session, fork and switch to the new conversation.
|
||||
pub(crate) async fn on_conversation_history_for_backtrack(
|
||||
&mut self,
|
||||
tui: &mut tui::Tui,
|
||||
ev: ConversationHistoryResponseEvent,
|
||||
) -> Result<()> {
|
||||
if let Some((base_id, _, _)) = self.backtrack.pending.as_ref()
|
||||
&& ev.conversation_id == *base_id
|
||||
&& let Some((_, drop_count, prefill)) = self.backtrack.pending.take()
|
||||
{
|
||||
self.fork_and_switch_to_new_conversation(tui, ev, drop_count, prefill)
|
||||
.await;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Fork the conversation using provided history and switch UI/state accordingly.
|
||||
async fn fork_and_switch_to_new_conversation(
|
||||
&mut self,
|
||||
tui: &mut tui::Tui,
|
||||
ev: ConversationHistoryResponseEvent,
|
||||
drop_count: usize,
|
||||
prefill: String,
|
||||
) {
|
||||
let cfg = self.chat_widget.config_ref().clone();
|
||||
// Perform the fork via a thin wrapper for clarity/testability.
|
||||
let result = self
|
||||
.perform_fork(ev.entries.clone(), drop_count, cfg.clone())
|
||||
.await;
|
||||
match result {
|
||||
Ok(new_conv) => {
|
||||
self.install_forked_conversation(tui, cfg, new_conv, drop_count, &prefill)
|
||||
}
|
||||
Err(e) => tracing::error!("error forking conversation: {e:#}"),
|
||||
}
|
||||
}
|
||||
|
||||
/// Thin wrapper around ConversationManager::fork_conversation.
|
||||
async fn perform_fork(
|
||||
&self,
|
||||
entries: Vec<codex_protocol::models::ResponseItem>,
|
||||
drop_count: usize,
|
||||
cfg: codex_core::config::Config,
|
||||
) -> codex_core::error::Result<codex_core::NewConversation> {
|
||||
self.server
|
||||
.fork_conversation(entries, drop_count, cfg)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Install a forked conversation into the ChatWidget and update UI to reflect selection.
|
||||
fn install_forked_conversation(
|
||||
&mut self,
|
||||
tui: &mut tui::Tui,
|
||||
cfg: codex_core::config::Config,
|
||||
new_conv: codex_core::NewConversation,
|
||||
drop_count: usize,
|
||||
prefill: &str,
|
||||
) {
|
||||
let conv = new_conv.conversation;
|
||||
let session_configured = new_conv.session_configured;
|
||||
self.chat_widget = crate::chatwidget::ChatWidget::new_from_existing(
|
||||
cfg,
|
||||
conv,
|
||||
session_configured,
|
||||
tui.frame_requester(),
|
||||
self.app_event_tx.clone(),
|
||||
self.enhanced_keys_supported,
|
||||
);
|
||||
// Trim transcript up to the selected user message and re-render it.
|
||||
self.trim_transcript_for_backtrack(drop_count);
|
||||
self.render_transcript_once(tui);
|
||||
if !prefill.is_empty() {
|
||||
self.chat_widget.insert_str(prefill);
|
||||
}
|
||||
tui.frame_requester().schedule_frame();
|
||||
}
|
||||
|
||||
/// Trim transcript_lines to preserve only content up to the selected user message.
|
||||
fn trim_transcript_for_backtrack(&mut self, drop_count: usize) {
|
||||
if let Some(cut_idx) =
|
||||
backtrack_helpers::find_nth_last_user_header_index(&self.transcript_lines, drop_count)
|
||||
{
|
||||
self.transcript_lines.truncate(cut_idx);
|
||||
} else {
|
||||
self.transcript_lines.clear();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,3 +1,4 @@
|
||||
use codex_core::protocol::ConversationHistoryResponseEvent;
|
||||
use codex_core::protocol::Event;
|
||||
use codex_file_search::FileMatch;
|
||||
use ratatui::text::Line;
|
||||
@@ -57,4 +58,7 @@ pub(crate) enum AppEvent {
|
||||
|
||||
/// Update the current sandbox policy in the running app and widget.
|
||||
UpdateSandboxPolicy(SandboxPolicy),
|
||||
|
||||
/// Forwarded conversation history snapshot from the current conversation.
|
||||
ConversationHistory(ConversationHistoryResponseEvent),
|
||||
}
|
||||
|
||||
154
codex-rs/tui/src/backtrack_helpers.rs
Normal file
154
codex-rs/tui/src/backtrack_helpers.rs
Normal file
@@ -0,0 +1,154 @@
|
||||
use ratatui::text::Line;
|
||||
|
||||
/// Convenience: compute the highlight range for the Nth last user message.
|
||||
pub(crate) fn highlight_range_for_nth_last_user(
|
||||
lines: &[Line<'_>],
|
||||
n: usize,
|
||||
) -> Option<(usize, usize)> {
|
||||
let header = find_nth_last_user_header_index(lines, n)?;
|
||||
Some(highlight_range_from_header(lines, header))
|
||||
}
|
||||
|
||||
/// Compute the wrapped display-line offset before `header_idx`, for a given width.
|
||||
pub(crate) fn wrapped_offset_before(lines: &[Line<'_>], header_idx: usize, width: u16) -> usize {
|
||||
let before = &lines[0..header_idx];
|
||||
crate::insert_history::word_wrap_lines(before, width).len()
|
||||
}
|
||||
|
||||
/// Find the header index for the Nth last user message in the transcript.
|
||||
/// Returns `None` if `n == 0` or there are fewer than `n` user messages.
|
||||
pub(crate) fn find_nth_last_user_header_index(lines: &[Line<'_>], n: usize) -> Option<usize> {
|
||||
if n == 0 {
|
||||
return None;
|
||||
}
|
||||
let mut found = 0usize;
|
||||
for (idx, line) in lines.iter().enumerate().rev() {
|
||||
let content: String = line
|
||||
.spans
|
||||
.iter()
|
||||
.map(|s| s.content.as_ref())
|
||||
.collect::<Vec<_>>()
|
||||
.join("");
|
||||
if content.trim() == "user" {
|
||||
found += 1;
|
||||
if found == n {
|
||||
return Some(idx);
|
||||
}
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Normalize a requested backtrack step `n` against the available user messages.
|
||||
/// - Returns `0` if there are no user messages.
|
||||
/// - Returns `n` if the Nth last user message exists.
|
||||
/// - Otherwise wraps to `1` (the most recent user message).
|
||||
pub(crate) fn normalize_backtrack_n(lines: &[Line<'_>], n: usize) -> usize {
|
||||
if n == 0 {
|
||||
return 0;
|
||||
}
|
||||
if find_nth_last_user_header_index(lines, n).is_some() {
|
||||
return n;
|
||||
}
|
||||
if find_nth_last_user_header_index(lines, 1).is_some() {
|
||||
1
|
||||
} else {
|
||||
0
|
||||
}
|
||||
}
|
||||
|
||||
/// Extract the text content of the Nth last user message.
|
||||
/// The message body is considered to be the lines following the "user" header
|
||||
/// until the first blank line.
|
||||
pub(crate) fn nth_last_user_text(lines: &[Line<'_>], n: usize) -> Option<String> {
|
||||
let header_idx = find_nth_last_user_header_index(lines, n)?;
|
||||
extract_message_text_after_header(lines, header_idx)
|
||||
}
|
||||
|
||||
/// Extract message text starting after `header_idx` until the first blank line.
|
||||
fn extract_message_text_after_header(lines: &[Line<'_>], header_idx: usize) -> Option<String> {
|
||||
let start = header_idx + 1;
|
||||
let mut out: Vec<String> = Vec::new();
|
||||
for line in lines.iter().skip(start) {
|
||||
let is_blank = line
|
||||
.spans
|
||||
.iter()
|
||||
.all(|s| s.content.as_ref().trim().is_empty());
|
||||
if is_blank {
|
||||
break;
|
||||
}
|
||||
let text: String = line
|
||||
.spans
|
||||
.iter()
|
||||
.map(|s| s.content.as_ref())
|
||||
.collect::<Vec<_>>()
|
||||
.join("");
|
||||
out.push(text);
|
||||
}
|
||||
if out.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(out.join("\n"))
|
||||
}
|
||||
}
|
||||
|
||||
/// Given a header index, return the inclusive range for the message block
|
||||
/// [header_idx, end) where end is the first blank line after the header or the
|
||||
/// end of the transcript.
|
||||
fn highlight_range_from_header(lines: &[Line<'_>], header_idx: usize) -> (usize, usize) {
|
||||
let mut end = header_idx + 1;
|
||||
while end < lines.len() {
|
||||
let is_blank = lines[end]
|
||||
.spans
|
||||
.iter()
|
||||
.all(|s| s.content.as_ref().trim().is_empty());
|
||||
if is_blank {
|
||||
break;
|
||||
}
|
||||
end += 1;
|
||||
}
|
||||
(header_idx, end)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use ratatui::text::Span;
|
||||
|
||||
fn line(s: &str) -> Line<'static> {
|
||||
Line::from(Span::raw(s.to_string()))
|
||||
}
|
||||
|
||||
fn transcript_with_users(count: usize) -> Vec<Line<'static>> {
|
||||
// Build a transcript with `count` user messages, each followed by one body line and a blank line.
|
||||
let mut v = Vec::new();
|
||||
for i in 0..count {
|
||||
v.push(line("user"));
|
||||
v.push(line(&format!("message {i}")));
|
||||
v.push(line(""));
|
||||
}
|
||||
v
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normalize_wraps_to_one_when_past_oldest() {
|
||||
let lines = transcript_with_users(2);
|
||||
assert_eq!(normalize_backtrack_n(&lines, 1), 1);
|
||||
assert_eq!(normalize_backtrack_n(&lines, 2), 2);
|
||||
// Requesting 3rd when only 2 exist wraps to 1
|
||||
assert_eq!(normalize_backtrack_n(&lines, 3), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normalize_returns_zero_when_no_user_messages() {
|
||||
let lines = transcript_with_users(0);
|
||||
assert_eq!(normalize_backtrack_n(&lines, 1), 0);
|
||||
assert_eq!(normalize_backtrack_n(&lines, 5), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normalize_keeps_valid_n() {
|
||||
let lines = transcript_with_users(3);
|
||||
assert_eq!(normalize_backtrack_n(&lines, 2), 2);
|
||||
}
|
||||
}
|
||||
@@ -28,16 +28,6 @@ pub(crate) trait BottomPaneView {
|
||||
/// Render the view: this will be displayed in place of the composer.
|
||||
fn render(&self, area: Rect, buf: &mut Buffer);
|
||||
|
||||
/// Update the status indicator animated header. Default no-op.
|
||||
fn update_status_header(&mut self, _header: String) {
|
||||
// no-op
|
||||
}
|
||||
|
||||
/// Called when task completes to check if the view should be hidden.
|
||||
fn should_hide_when_task_is_done(&mut self) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
/// Try to handle approval request; return the original value if not
|
||||
/// consumed.
|
||||
fn try_consume_approval_request(
|
||||
@@ -46,8 +36,4 @@ pub(crate) trait BottomPaneView {
|
||||
) -> Option<ApprovalRequest> {
|
||||
Some(request)
|
||||
}
|
||||
|
||||
/// Optional hook for views that expose a live status line. Views that do not
|
||||
/// support this can ignore the call.
|
||||
fn update_status_text(&mut self, _text: String) {}
|
||||
}
|
||||
|
||||
@@ -29,6 +29,8 @@ use crate::app_event::AppEvent;
|
||||
use crate::app_event_sender::AppEventSender;
|
||||
use crate::bottom_pane::textarea::TextArea;
|
||||
use crate::bottom_pane::textarea::TextAreaState;
|
||||
use crate::clipboard_paste::normalize_pasted_path;
|
||||
use crate::clipboard_paste::pasted_image_format;
|
||||
use codex_file_search::FileMatch;
|
||||
use std::cell::RefCell;
|
||||
use std::collections::HashMap;
|
||||
@@ -81,6 +83,7 @@ pub(crate) struct ChatComposer {
|
||||
app_event_tx: AppEventSender,
|
||||
history: ChatComposerHistory,
|
||||
ctrl_c_quit_hint: bool,
|
||||
esc_backtrack_hint: bool,
|
||||
use_shift_enter_hint: bool,
|
||||
dismissed_file_popup_token: Option<String>,
|
||||
current_file_query: Option<String>,
|
||||
@@ -121,6 +124,7 @@ impl ChatComposer {
|
||||
app_event_tx,
|
||||
history: ChatComposerHistory::new(),
|
||||
ctrl_c_quit_hint: false,
|
||||
esc_backtrack_hint: false,
|
||||
use_shift_enter_hint,
|
||||
dismissed_file_popup_token: None,
|
||||
current_file_query: None,
|
||||
@@ -140,7 +144,13 @@ impl ChatComposer {
|
||||
pub fn desired_height(&self, width: u16) -> u16 {
|
||||
self.textarea.desired_height(width - 1)
|
||||
+ match &self.active_popup {
|
||||
ActivePopup::None => 1u16,
|
||||
ActivePopup::None => {
|
||||
if self.token_usage_info.is_some() && width < 90 {
|
||||
2
|
||||
} else {
|
||||
1
|
||||
}
|
||||
}
|
||||
ActivePopup::Command(c) => c.calculate_required_height(),
|
||||
ActivePopup::File(c) => c.calculate_required_height(),
|
||||
}
|
||||
@@ -150,10 +160,16 @@ impl ChatComposer {
|
||||
let popup_height = match &self.active_popup {
|
||||
ActivePopup::Command(popup) => popup.calculate_required_height(),
|
||||
ActivePopup::File(popup) => popup.calculate_required_height(),
|
||||
ActivePopup::None => 1,
|
||||
ActivePopup::None => {
|
||||
if self.token_usage_info.is_some() && area.width < 90 {
|
||||
2
|
||||
} else {
|
||||
1
|
||||
}
|
||||
}
|
||||
};
|
||||
let [textarea_rect, _] =
|
||||
Layout::vertical([Constraint::Min(0), Constraint::Max(popup_height)]).areas(area);
|
||||
Layout::vertical([Constraint::Min(1), Constraint::Max(popup_height)]).areas(area);
|
||||
let mut textarea_rect = textarea_rect;
|
||||
textarea_rect.width = textarea_rect.width.saturating_sub(1);
|
||||
textarea_rect.x += 1;
|
||||
@@ -218,6 +234,8 @@ impl ChatComposer {
|
||||
let placeholder = format!("[Pasted Content {char_count} chars]");
|
||||
self.textarea.insert_element(&placeholder);
|
||||
self.pending_pastes.push((placeholder, pasted));
|
||||
} else if self.handle_paste_image_path(pasted.clone()) {
|
||||
self.textarea.insert_str(" ");
|
||||
} else {
|
||||
self.textarea.insert_str(&pasted);
|
||||
}
|
||||
@@ -230,6 +248,39 @@ impl ChatComposer {
|
||||
true
|
||||
}
|
||||
|
||||
pub fn handle_paste_image_path(&mut self, pasted: String) -> bool {
|
||||
let Some(path_buf) = normalize_pasted_path(&pasted) else {
|
||||
return false;
|
||||
};
|
||||
|
||||
match image::image_dimensions(&path_buf) {
|
||||
Ok((w, h)) => {
|
||||
tracing::info!("OK: {pasted}");
|
||||
let format_label = pasted_image_format(&path_buf).label();
|
||||
self.attach_image(path_buf, w, h, format_label);
|
||||
true
|
||||
}
|
||||
Err(err) => {
|
||||
tracing::info!("ERR: {err}");
|
||||
false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Replace the entire composer content with `text` and reset cursor.
|
||||
pub(crate) fn set_text_content(&mut self, text: String) {
|
||||
self.textarea.set_text(&text);
|
||||
self.textarea.set_cursor(0);
|
||||
self.sync_command_popup();
|
||||
self.sync_file_search_popup();
|
||||
}
|
||||
|
||||
/// Get the current composer text.
|
||||
#[cfg(test)]
|
||||
pub(crate) fn current_text(&self) -> String {
|
||||
self.textarea.text().to_string()
|
||||
}
|
||||
|
||||
pub fn attach_image(&mut self, path: PathBuf, width: u32, height: u32, format_label: &str) {
|
||||
let placeholder = format!("[image {width}x{height} {format_label}]");
|
||||
// Insert as an element to match large paste placeholder behavior:
|
||||
@@ -291,6 +342,11 @@ impl ChatComposer {
|
||||
result
|
||||
}
|
||||
|
||||
/// Return true if either the slash-command popup or the file-search popup is active.
|
||||
pub(crate) fn popup_active(&self) -> bool {
|
||||
!matches!(self.active_popup, ActivePopup::None)
|
||||
}
|
||||
|
||||
/// Handle key event when the slash-command popup is visible.
|
||||
fn handle_key_event_with_slash_popup(&mut self, key_event: KeyEvent) -> (InputResult, bool) {
|
||||
let ActivePopup::Command(popup) = &mut self.active_popup else {
|
||||
@@ -311,6 +367,13 @@ impl ChatComposer {
|
||||
popup.move_down();
|
||||
(InputResult::None, true)
|
||||
}
|
||||
KeyEvent {
|
||||
code: KeyCode::Esc, ..
|
||||
} => {
|
||||
// Dismiss the slash popup; keep the current input untouched.
|
||||
self.active_popup = ActivePopup::None;
|
||||
(InputResult::None, true)
|
||||
}
|
||||
KeyEvent {
|
||||
code: KeyCode::Tab, ..
|
||||
} => {
|
||||
@@ -702,13 +765,6 @@ impl ChatComposer {
|
||||
}
|
||||
self.pending_pastes.clear();
|
||||
|
||||
// Strip image placeholders from the submitted text; images are retrieved via take_recent_submission_images()
|
||||
for img in &self.attached_images {
|
||||
if text.contains(&img.placeholder) {
|
||||
text = text.replace(&img.placeholder, "");
|
||||
}
|
||||
}
|
||||
|
||||
text = text.trim().to_string();
|
||||
if !text.is_empty() {
|
||||
self.history.record_local_submission(&text);
|
||||
@@ -1091,17 +1147,29 @@ impl ChatComposer {
|
||||
fn set_has_focus(&mut self, has_focus: bool) {
|
||||
self.has_focus = has_focus;
|
||||
}
|
||||
|
||||
pub(crate) fn set_esc_backtrack_hint(&mut self, show: bool) {
|
||||
self.esc_backtrack_hint = show;
|
||||
}
|
||||
}
|
||||
|
||||
impl WidgetRef for &ChatComposer {
|
||||
impl WidgetRef for ChatComposer {
|
||||
fn render_ref(&self, area: Rect, buf: &mut Buffer) {
|
||||
// Determine if we should render a second footer row for tokens/context.
|
||||
let needs_tokens_line = self.token_usage_info.is_some() && area.width < 90;
|
||||
let popup_height = match &self.active_popup {
|
||||
ActivePopup::Command(popup) => popup.calculate_required_height(),
|
||||
ActivePopup::File(popup) => popup.calculate_required_height(),
|
||||
ActivePopup::None => 1,
|
||||
ActivePopup::None => {
|
||||
if needs_tokens_line {
|
||||
2
|
||||
} else {
|
||||
1
|
||||
}
|
||||
}
|
||||
};
|
||||
let [textarea_rect, popup_rect] =
|
||||
Layout::vertical([Constraint::Min(0), Constraint::Max(popup_height)]).areas(area);
|
||||
Layout::vertical([Constraint::Min(1), Constraint::Max(popup_height)]).areas(area);
|
||||
match &self.active_popup {
|
||||
ActivePopup::Command(popup) => {
|
||||
popup.render_ref(popup_rect, buf);
|
||||
@@ -1130,13 +1198,20 @@ impl WidgetRef for &ChatComposer {
|
||||
Span::from(" send "),
|
||||
newline_hint_key.set_style(key_hint_style),
|
||||
Span::from(" newline "),
|
||||
"Ctrl+T".set_style(key_hint_style),
|
||||
Span::from(" transcript "),
|
||||
"Ctrl+C".set_style(key_hint_style),
|
||||
Span::from(" quit"),
|
||||
]
|
||||
};
|
||||
|
||||
// Append token/context usage info to the footer hints when available.
|
||||
if let Some(token_usage_info) = &self.token_usage_info {
|
||||
if !self.ctrl_c_quit_hint && self.esc_backtrack_hint {
|
||||
hint.push(Span::from(" "));
|
||||
hint.push("Esc".set_style(key_hint_style));
|
||||
hint.push(Span::from(" edit prev"));
|
||||
}
|
||||
|
||||
if !needs_tokens_line && let Some(token_usage_info) = &self.token_usage_info {
|
||||
let token_usage = &token_usage_info.total_token_usage;
|
||||
hint.push(Span::from(" "));
|
||||
hint.push(
|
||||
@@ -1164,6 +1239,34 @@ impl WidgetRef for &ChatComposer {
|
||||
Line::from(hint)
|
||||
.style(Style::default().dim())
|
||||
.render_ref(bottom_line_rect, buf);
|
||||
|
||||
if needs_tokens_line && let Some(token_usage_info) = &self.token_usage_info {
|
||||
let token_usage = &token_usage_info.total_token_usage;
|
||||
let mut text = String::from(" ");
|
||||
text.push_str(&format!("{} tokens used", token_usage.blended_total()));
|
||||
if let Some(context_window) = token_usage_info.model_context_window {
|
||||
let last_token_usage = &token_usage_info.last_token_usage;
|
||||
let percent_remaining: u8 = if context_window > 0 {
|
||||
last_token_usage.percent_of_context_window_remaining(
|
||||
context_window,
|
||||
token_usage_info.initial_prompt_tokens,
|
||||
)
|
||||
} else {
|
||||
100
|
||||
};
|
||||
text.push_str(" ");
|
||||
text.push_str(&format!("{percent_remaining}% context left"));
|
||||
}
|
||||
Line::from(text).style(Style::default().dim()).render_ref(
|
||||
Rect::new(
|
||||
bottom_line_rect.x,
|
||||
bottom_line_rect.y + 1,
|
||||
bottom_line_rect.width,
|
||||
1,
|
||||
),
|
||||
buf,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
let border_style = if self.has_focus {
|
||||
@@ -1196,7 +1299,10 @@ impl WidgetRef for &ChatComposer {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use image::ImageBuffer;
|
||||
use image::Rgba;
|
||||
use std::path::PathBuf;
|
||||
use tempfile::tempdir;
|
||||
|
||||
use crate::app_event::AppEvent;
|
||||
use crate::bottom_pane::AppEventSender;
|
||||
@@ -1482,7 +1588,7 @@ mod tests {
|
||||
}
|
||||
|
||||
terminal
|
||||
.draw(|f| f.render_widget_ref(&composer, f.area()))
|
||||
.draw(|f| f.render_widget_ref(composer, f.area()))
|
||||
.unwrap_or_else(|e| panic!("Failed to draw {name} composer: {e}"));
|
||||
|
||||
assert_snapshot!(name, terminal.backend());
|
||||
@@ -1779,7 +1885,7 @@ mod tests {
|
||||
let (result, _) =
|
||||
composer.handle_key_event(KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE));
|
||||
match result {
|
||||
InputResult::Submitted(text) => assert_eq!(text, "hi"),
|
||||
InputResult::Submitted(text) => assert_eq!(text, "[image 32x16 PNG] hi"),
|
||||
_ => panic!("expected Submitted"),
|
||||
}
|
||||
let imgs = composer.take_recent_submission_images();
|
||||
@@ -1797,7 +1903,7 @@ mod tests {
|
||||
let (result, _) =
|
||||
composer.handle_key_event(KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE));
|
||||
match result {
|
||||
InputResult::Submitted(text) => assert!(text.is_empty()),
|
||||
InputResult::Submitted(text) => assert_eq!(text, "[image 10x5 PNG]"),
|
||||
_ => panic!("expected Submitted"),
|
||||
}
|
||||
let imgs = composer.take_recent_submission_images();
|
||||
@@ -1873,4 +1979,25 @@ mod tests {
|
||||
"one image mapping remains"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pasting_filepath_attaches_image() {
|
||||
let tmp = tempdir().expect("create TempDir");
|
||||
let tmp_path: PathBuf = tmp.path().join("codex_tui_test_paste_image.png");
|
||||
let img: ImageBuffer<Rgba<u8>, Vec<u8>> =
|
||||
ImageBuffer::from_fn(3, 2, |_x, _y| Rgba([1, 2, 3, 255]));
|
||||
img.save(&tmp_path).expect("failed to write temp png");
|
||||
|
||||
let (tx, _rx) = unbounded_channel::<AppEvent>();
|
||||
let sender = AppEventSender::new(tx);
|
||||
let mut composer =
|
||||
ChatComposer::new(true, sender, false, "Ask Codex to do anything".to_string());
|
||||
|
||||
let needs_redraw = composer.handle_paste(tmp_path.to_string_lossy().to_string());
|
||||
assert!(needs_redraw);
|
||||
assert!(composer.textarea.text().starts_with("[image 3x2 PNG] "));
|
||||
|
||||
let imgs = composer.take_recent_submission_images();
|
||||
assert_eq!(imgs, vec![tmp_path.clone()]);
|
||||
}
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user