Compare commits

..

15 Commits

Author SHA1 Message Date
Ahmed Ibrahim
257d5eceff use shared tokenizer 2025-10-24 12:31:08 -07:00
Ahmed Ibrahim
56dfa60801 use shared tokenizer 2025-10-24 12:05:06 -07:00
Ahmed Ibrahim
50055c7ed5 use shared tokenizer 2025-10-24 12:02:48 -07:00
Ahmed Ibrahim
89e9bb1d42 comment 2025-10-24 11:44:05 -07:00
Ahmed Ibrahim
81d0003f0f tests 2025-10-24 11:36:36 -07:00
Ahmed Ibrahim
5d7983520b tests 2025-10-24 11:23:25 -07:00
Ahmed Ibrahim
4e7089a8ab default 2025-10-24 11:16:45 -07:00
Ahmed Ibrahim
697367cd3f bug 2025-10-24 10:12:36 -07:00
Ahmed Ibrahim
6ffbd0d4e3 bug 2025-10-24 10:08:47 -07:00
Ahmed Ibrahim
79c628a823 tests 2025-10-24 09:55:08 -07:00
Ahmed Ibrahim
9446de0923 tests 2025-10-23 23:26:02 -07:00
Ahmed Ibrahim
0f02954edb Merge branch 'main' into input-validation 2025-10-23 22:36:12 -07:00
Ahmed Ibrahim
7a8da22d7e add file 2025-10-23 19:03:24 -07:00
Ahmed Ibrahim
8f8ca17da0 input-validation 2025-10-23 18:58:08 -07:00
Ahmed Ibrahim
8b095d3cf1 add-input-validation 2025-10-23 16:17:10 -07:00
41 changed files with 533 additions and 891 deletions

View File

@@ -33,8 +33,6 @@ Then simply run `codex` to get started:
codex
```
If you're running into upgrade issues with Homebrew, see the [FAQ entry on brew upgrade codex](./docs/faq.md#brew-update-codex-isnt-upgrading-me).
<details>
<summary>You can also go to the <a href="https://github.com/openai/codex/releases/latest">latest GitHub Release</a> and download the appropriate binary for your platform.</summary>

1
codex-rs/Cargo.lock generated
View File

@@ -1075,7 +1075,6 @@ dependencies = [
"escargot",
"eventsource-stream",
"futures",
"http",
"indexmap 2.10.0",
"landlock",
"libc",

View File

@@ -116,7 +116,6 @@ env_logger = "0.11.5"
escargot = "0.5"
eventsource-stream = "0.2.3"
futures = { version = "0.3", default-features = false }
http = "1.3.1"
icu_decimal = "2.0.0"
icu_locale_core = "2.0.0"
ignore = "0.4.23"

View File

@@ -23,7 +23,6 @@ use std::io::Write;
use std::path::Path;
use std::path::PathBuf;
use std::process::Command;
use ts_rs::ExportError;
use ts_rs::TS;
const HEADER: &str = "// GENERATED CODE! DO NOT MODIFY BY HAND!\n\n";
@@ -105,19 +104,6 @@ macro_rules! for_each_schema_type {
};
}
fn export_ts_with_context<F>(label: &str, export: F) -> Result<()>
where
F: FnOnce() -> std::result::Result<(), ExportError>,
{
match export() {
Ok(()) => Ok(()),
Err(ExportError::CannotBeExported(ty)) => Err(anyhow!(
"failed to export {label}: dependency {ty} cannot be exported"
)),
Err(err) => Err(err.into()),
}
}
pub fn generate_types(out_dir: &Path, prettier: Option<&Path>) -> Result<()> {
generate_ts(out_dir, prettier)?;
generate_json(out_dir)?;
@@ -127,17 +113,13 @@ pub fn generate_types(out_dir: &Path, prettier: Option<&Path>) -> Result<()> {
pub fn generate_ts(out_dir: &Path, prettier: Option<&Path>) -> Result<()> {
ensure_dir(out_dir)?;
export_ts_with_context("ClientRequest", || ClientRequest::export_all_to(out_dir))?;
export_ts_with_context("client responses", || export_client_responses(out_dir))?;
export_ts_with_context("ClientNotification", || {
ClientNotification::export_all_to(out_dir)
})?;
ClientRequest::export_all_to(out_dir)?;
export_client_responses(out_dir)?;
ClientNotification::export_all_to(out_dir)?;
export_ts_with_context("ServerRequest", || ServerRequest::export_all_to(out_dir))?;
export_ts_with_context("server responses", || export_server_responses(out_dir))?;
export_ts_with_context("ServerNotification", || {
ServerNotification::export_all_to(out_dir)
})?;
ServerRequest::export_all_to(out_dir)?;
export_server_responses(out_dir)?;
ServerNotification::export_all_to(out_dir)?;
generate_index_ts(out_dir)?;

View File

@@ -127,7 +127,7 @@ client_request_definitions! {
#[ts(rename = "account/read")]
GetAccount {
params: #[ts(type = "undefined")] #[serde(skip_serializing_if = "Option::is_none")] Option<()>,
response: GetAccountResponse,
response: Option<Account>,
},
/// DEPRECATED APIs below
@@ -534,12 +534,6 @@ pub struct GetAccountRateLimitsResponse {
pub rate_limits: RateLimitSnapshot,
}
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)]
#[serde(transparent)]
#[ts(export)]
#[ts(type = "Account | null")]
pub struct GetAccountResponse(#[ts(type = "Account | null")] pub Option<Account>);
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)]
#[serde(rename_all = "camelCase")]
pub struct GetAuthStatusResponse {

View File

@@ -34,7 +34,6 @@ dunce = { workspace = true }
env-flags = { workspace = true }
eventsource-stream = { workspace = true }
futures = { workspace = true }
http = { workspace = true }
indexmap = { workspace = true }
libc = { workspace = true }
mcp-types = { workspace = true }

View File

@@ -21,7 +21,6 @@ use codex_app_server_protocol::AuthMode;
use codex_protocol::config_types::ForcedLoginMethod;
use crate::config::Config;
use crate::default_client::CodexHttpClient;
use crate::token_data::PlanType;
use crate::token_data::TokenData;
use crate::token_data::parse_id_token;
@@ -33,7 +32,7 @@ pub struct CodexAuth {
pub(crate) api_key: Option<String>,
pub(crate) auth_dot_json: Arc<Mutex<Option<AuthDotJson>>>,
pub(crate) auth_file: PathBuf,
pub(crate) client: CodexHttpClient,
pub(crate) client: reqwest::Client,
}
impl PartialEq for CodexAuth {
@@ -44,8 +43,6 @@ impl PartialEq for CodexAuth {
impl CodexAuth {
pub async fn refresh_token(&self) -> Result<String, std::io::Error> {
tracing::info!("Refreshing token");
let token_data = self
.get_current_token_data()
.ok_or(std::io::Error::other("Token data is not available."))?;
@@ -183,7 +180,7 @@ impl CodexAuth {
}
}
fn from_api_key_with_client(api_key: &str, client: CodexHttpClient) -> Self {
fn from_api_key_with_client(api_key: &str, client: reqwest::Client) -> Self {
Self {
api_key: Some(api_key.to_owned()),
mode: AuthMode::ApiKey,
@@ -403,7 +400,7 @@ async fn update_tokens(
async fn try_refresh_token(
refresh_token: String,
client: &CodexHttpClient,
client: &reqwest::Client,
) -> std::io::Result<RefreshResponse> {
let refresh_request = RefreshRequest {
client_id: CLIENT_ID,
@@ -919,10 +916,7 @@ impl AuthManager {
self.reload();
Ok(Some(token))
}
Err(e) => {
tracing::error!("Failed to refresh token: {}", e);
Err(e)
}
Err(e) => Err(e),
}
}

View File

@@ -4,7 +4,6 @@ use crate::ModelProviderInfo;
use crate::client_common::Prompt;
use crate::client_common::ResponseEvent;
use crate::client_common::ResponseStream;
use crate::default_client::CodexHttpClient;
use crate::error::CodexErr;
use crate::error::ConnectionFailedError;
use crate::error::ResponseStreamFailed;
@@ -37,7 +36,7 @@ use tracing::trace;
pub(crate) async fn stream_chat_completions(
prompt: &Prompt,
model_family: &ModelFamily,
client: &CodexHttpClient,
client: &reqwest::Client,
provider: &ModelProviderInfo,
otel_event_manager: &OtelEventManager,
) -> Result<ResponseStream> {

View File

@@ -39,7 +39,6 @@ use crate::client_common::ResponsesApiRequest;
use crate::client_common::create_reasoning_param_for_request;
use crate::client_common::create_text_param_for_request;
use crate::config::Config;
use crate::default_client::CodexHttpClient;
use crate::default_client::create_client;
use crate::error::CodexErr;
use crate::error::ConnectionFailedError;
@@ -82,7 +81,7 @@ pub struct ModelClient {
config: Arc<Config>,
auth_manager: Option<Arc<AuthManager>>,
otel_event_manager: OtelEventManager,
client: CodexHttpClient,
client: reqwest::Client,
provider: ModelProviderInfo,
conversation_id: ConversationId,
effort: Option<ReasoningEffortConfig>,
@@ -301,7 +300,6 @@ impl ModelClient {
"POST to {}: {:?}",
self.provider.get_full_url(&auth),
serde_json::to_string(payload_json)
.unwrap_or("<unable to serialize payload>".to_string())
);
let mut req_builder = self
@@ -337,6 +335,13 @@ impl ModelClient {
.headers()
.get("cf-ray")
.map(|v| v.to_str().unwrap_or_default().to_string());
debug!(
"Response status: {}, cf-ray: {:?}, version: {:?}",
resp.status(),
request_id,
resp.version()
);
}
match res {

View File

@@ -8,7 +8,6 @@ use crate::AuthManager;
use crate::client_common::REVIEW_PROMPT;
use crate::function_tool::FunctionCallError;
use crate::mcp::auth::McpAuthStatusEntry;
use crate::mcp_connection_manager::DEFAULT_STARTUP_TIMEOUT;
use crate::parse_command::parse_command;
use crate::parse_turn_item;
use crate::response_processing::process_items;
@@ -59,7 +58,8 @@ use crate::client_common::ResponseEvent;
use crate::config::Config;
use crate::config_types::McpServerTransportConfig;
use crate::config_types::ShellEnvironmentPolicy;
use crate::context_manager::ContextManager;
use crate::conversation_history::ConversationHistory;
use crate::conversation_history::prefetch_tokenizer_in_background;
use crate::environment_context::EnvironmentContext;
use crate::error::CodexErr;
use crate::error::Result as CodexResult;
@@ -160,6 +160,8 @@ impl Codex {
conversation_history: InitialHistory,
session_source: SessionSource,
) -> CodexResult<CodexSpawnOk> {
// Start loading the tokenizer in the background so we don't block later.
prefetch_tokenizer_in_background();
let (tx_sub, rx_sub) = async_channel::bounded(SUBMISSION_CHANNEL_CAPACITY);
let (tx_event, rx_event) = async_channel::unbounded();
@@ -569,7 +571,9 @@ impl Session {
// Dispatch the SessionConfiguredEvent first and then report any errors.
// If resuming, include converted initial messages in the payload so UIs can render them immediately.
let initial_messages = initial_history.get_event_msgs();
sess.record_initial_history(initial_history).await;
sess.record_initial_history(initial_history)
.await
.map_err(anyhow::Error::new)?;
let events = std::iter::once(Event {
id: INITIAL_SUBMIT_ID.to_owned(),
@@ -602,13 +606,16 @@ impl Session {
format!("auto-compact-{id}")
}
async fn record_initial_history(&self, conversation_history: InitialHistory) {
async fn record_initial_history(
&self,
conversation_history: InitialHistory,
) -> CodexResult<()> {
let turn_context = self.new_turn(SessionSettingsUpdate::default()).await;
match conversation_history {
InitialHistory::New => {
// Build and record initial items (user instructions + environment context)
let items = self.build_initial_context(&turn_context);
self.record_conversation_items(&items).await;
self.record_conversation_items(&items).await?;
}
InitialHistory::Resumed(_) | InitialHistory::Forked(_) => {
let rollout_items = conversation_history.get_rollout_items();
@@ -616,9 +623,9 @@ impl Session {
// Always add response items to conversation history
let reconstructed_history =
self.reconstruct_history_from_rollout(&turn_context, &rollout_items);
self.reconstruct_history_from_rollout(&turn_context, &rollout_items)?;
if !reconstructed_history.is_empty() {
self.record_into_history(&reconstructed_history).await;
self.record_into_history(&reconstructed_history).await?;
}
// If persisting, persist all rollout items as-is (recorder filters)
@@ -627,6 +634,7 @@ impl Session {
}
}
}
Ok(())
}
pub(crate) async fn update_settings(&self, updates: SessionSettingsUpdate) {
@@ -857,21 +865,25 @@ impl Session {
/// Records input items: always append to conversation history and
/// persist these response items to rollout.
pub(crate) async fn record_conversation_items(&self, items: &[ResponseItem]) {
self.record_into_history(items).await;
pub(crate) async fn record_conversation_items(
&self,
items: &[ResponseItem],
) -> CodexResult<()> {
self.record_into_history(items).await?;
self.persist_rollout_response_items(items).await;
Ok(())
}
fn reconstruct_history_from_rollout(
&self,
turn_context: &TurnContext,
rollout_items: &[RolloutItem],
) -> Vec<ResponseItem> {
let mut history = ContextManager::new();
) -> CodexResult<Vec<ResponseItem>> {
let mut history = ConversationHistory::new();
for item in rollout_items {
match item {
RolloutItem::ResponseItem(response_item) => {
history.record_items(std::iter::once(response_item));
history.record_items(std::iter::once(response_item))?;
}
RolloutItem::Compacted(compacted) => {
let snapshot = history.get_history();
@@ -886,13 +898,14 @@ impl Session {
_ => {}
}
}
history.get_history()
Ok(history.get_history())
}
/// Append ResponseItems to the in-memory conversation history only.
async fn record_into_history(&self, items: &[ResponseItem]) {
async fn record_into_history(&self, items: &[ResponseItem]) -> CodexResult<()> {
let mut state = self.state.lock().await;
state.record_items(items.iter());
state.record_items(items.iter())?;
Ok(())
}
async fn replace_history(&self, items: Vec<ResponseItem>) {
@@ -941,7 +954,7 @@ impl Session {
state.history_snapshot()
}
pub(crate) async fn clone_history(&self) -> ContextManager {
pub(crate) async fn clone_history(&self) -> ConversationHistory {
let state = self.state.lock().await;
state.clone_history()
}
@@ -1001,11 +1014,11 @@ impl Session {
&self,
turn_context: &TurnContext,
response_input: &ResponseInputItem,
) {
) -> CodexResult<()> {
let response_item: ResponseItem = response_input.clone().into();
// Add to conversation history and persist response item to rollout
self.record_conversation_items(std::slice::from_ref(&response_item))
.await;
.await?;
// Derive user message events and persist only UserMessage to rollout
let turn_item = parse_turn_item(&response_item);
@@ -1014,6 +1027,7 @@ impl Session {
self.emit_turn_item_started_completed(turn_context, item, false)
.await;
}
Ok(())
}
/// Helper that emits a BackgroundEvent with the given message. This keeps
@@ -1194,9 +1208,17 @@ async fn submission_loop(sess: Arc<Session>, config: Arc<Config>, rx_sub: Receiv
if let Err(items) = sess.inject_input(items).await {
if let Some(env_item) = sess
.build_environment_update_item(previous_context.as_ref(), &current_context)
&& let Err(err) = sess
.record_conversation_items(std::slice::from_ref(&env_item))
.await
{
sess.record_conversation_items(std::slice::from_ref(&env_item))
.await;
sess.send_event(
current_context.as_ref(),
EventMsg::Error(ErrorEvent {
message: err.to_string(),
}),
)
.await;
}
sess.spawn_task(Arc::clone(&current_context), items, RegularTask)
@@ -1509,9 +1531,9 @@ pub(crate) async fn run_task(
input: Vec<UserInput>,
task_kind: TaskKind,
cancellation_token: CancellationToken,
) -> Option<String> {
) -> CodexResult<Option<String>> {
if input.is_empty() {
return None;
return Ok(None);
}
let event = EventMsg::TaskStarted(TaskStartedEvent {
model_context_window: turn_context.client.get_model_context_window(),
@@ -1524,15 +1546,15 @@ pub(crate) async fn run_task(
// For normal turns, continue recording to the session history as before.
let is_review_mode = turn_context.is_review_mode;
let mut review_thread_history: ContextManager = ContextManager::new();
let mut review_thread_history: ConversationHistory = ConversationHistory::new();
if is_review_mode {
// Seed review threads with environment context so the model knows the working directory.
review_thread_history
.record_items(sess.build_initial_context(turn_context.as_ref()).iter());
review_thread_history.record_items(std::iter::once(&initial_input_for_turn.into()));
.record_items(sess.build_initial_context(turn_context.as_ref()).iter())?;
review_thread_history.record_items(std::iter::once(&initial_input_for_turn.into()))?;
} else {
sess.record_input_and_rollout_usermsg(turn_context.as_ref(), &initial_input_for_turn)
.await;
.await?;
}
let mut last_agent_message: Option<String> = None;
@@ -1564,11 +1586,11 @@ pub(crate) async fn run_task(
// represents an append-only log without duplicates.
let turn_input: Vec<ResponseItem> = if is_review_mode {
if !pending_input.is_empty() {
review_thread_history.record_items(&pending_input);
review_thread_history.record_items(&pending_input)?;
}
review_thread_history.get_history()
} else {
sess.record_conversation_items(&pending_input).await;
sess.record_conversation_items(&pending_input).await?;
sess.history_snapshot().await
};
@@ -1616,7 +1638,7 @@ pub(crate) async fn run_task(
&mut review_thread_history,
&sess,
)
.await;
.await?;
if token_limit_reached {
if auto_compact_recently_attempted {
@@ -1633,7 +1655,8 @@ pub(crate) async fn run_task(
break;
}
auto_compact_recently_attempted = true;
compact::run_inline_auto_compact_task(sess.clone(), turn_context.clone()).await;
compact::run_inline_auto_compact_task(sess.clone(), turn_context.clone())
.await?;
continue;
}
@@ -1658,13 +1681,13 @@ pub(crate) async fn run_task(
Err(CodexErr::TurnAborted {
dangling_artifacts: processed_items,
}) => {
let _ = process_items(
process_items(
processed_items,
is_review_mode,
&mut review_thread_history,
&sess,
)
.await;
.await?;
// Aborted turn is reported via a different event.
break;
}
@@ -1693,10 +1716,10 @@ pub(crate) async fn run_task(
Arc::clone(&turn_context),
last_agent_message.as_deref().map(parse_review_output_event),
)
.await;
.await?;
}
last_agent_message
Ok(last_agent_message)
}
/// Parse the review output; when not valid JSON, build a structured
@@ -2135,7 +2158,7 @@ pub(crate) async fn exit_review_mode(
session: Arc<Session>,
turn_context: Arc<TurnContext>,
review_output: Option<ReviewOutputEvent>,
) {
) -> CodexResult<()> {
let event = EventMsg::ExitedReviewMode(ExitedReviewModeEvent {
review_output: review_output.clone(),
});
@@ -2178,7 +2201,8 @@ pub(crate) async fn exit_review_mode(
role: "user".to_string(),
content: vec![ContentItem::InputText { text: user_message }],
}])
.await;
.await?;
Ok(())
}
fn mcp_init_error_display(
@@ -2200,24 +2224,12 @@ fn mcp_init_error_display(
// That means that the user has to specify a personal access token either via bearer_token_env_var or http_headers.
// https://github.com/github/github-mcp-server/issues/921#issuecomment-3221026448
format!(
"GitHub MCP does not support OAuth. Log in by adding a personal access token (https://github.com/settings/personal-access-tokens) to your environment and config.toml:\n[mcp_servers.{server_name}]\nbearer_token_env_var = CODEX_GITHUB_PERSONAL_ACCESS_TOKEN"
"GitHub MCP does not support OAuth. Log in by adding `bearer_token_env_var = CODEX_GITHUB_PAT` in the `mcp_servers.{server_name}` section of your config.toml"
)
} else if is_mcp_client_auth_required_error(err) {
format!(
"The {server_name} MCP server is not logged in. Run `codex mcp login {server_name}`."
)
} else if is_mcp_client_startup_timeout_error(err) {
let startup_timeout_secs = match entry {
Some(entry) => match entry.config.startup_timeout_sec {
Some(timeout) => timeout,
None => DEFAULT_STARTUP_TIMEOUT,
},
None => DEFAULT_STARTUP_TIMEOUT,
}
.as_secs();
format!(
"MCP client for `{server_name}` timed out after {startup_timeout_secs} seconds. Add or adjust `startup_timeout_sec` in your config.toml:\n[mcp_servers.{server_name}]\nstartup_timeout_sec = XX"
)
} else {
format!("MCP client for `{server_name}` failed to start: {err:#}")
}
@@ -2228,12 +2240,6 @@ fn is_mcp_client_auth_required_error(error: &anyhow::Error) -> bool {
error.to_string().contains("Auth required")
}
fn is_mcp_client_startup_timeout_error(error: &anyhow::Error) -> bool {
let error_message = error.to_string();
error_message.contains("request timed out")
|| error_message.contains("timed out handshaking with MCP server")
}
#[cfg(test)]
pub(crate) use tests::make_session_and_context;
@@ -2244,6 +2250,7 @@ mod tests {
use crate::config::ConfigToml;
use crate::config_types::McpServerConfig;
use crate::config_types::McpServerTransportConfig;
use crate::error::Result as CodexResult;
use crate::exec::ExecToolCallOutput;
use crate::mcp::auth::McpAuthStatusEntry;
use crate::tools::format_exec_output_str;
@@ -2284,9 +2291,12 @@ mod tests {
#[test]
fn reconstruct_history_matches_live_compactions() {
let (session, turn_context) = make_session_and_context();
let (rollout_items, expected) = sample_rollout(&session, &turn_context);
let (rollout_items, expected) =
sample_rollout(&session, &turn_context).expect("sample rollout");
let reconstructed = session.reconstruct_history_from_rollout(&turn_context, &rollout_items);
let reconstructed = session
.reconstruct_history_from_rollout(&turn_context, &rollout_items)
.expect("reconstruct history");
assert_eq!(expected, reconstructed);
}
@@ -2294,15 +2304,19 @@ mod tests {
#[test]
fn record_initial_history_reconstructs_resumed_transcript() {
let (session, turn_context) = make_session_and_context();
let (rollout_items, expected) = sample_rollout(&session, &turn_context);
let (rollout_items, expected) =
sample_rollout(&session, &turn_context).expect("sample rollout");
tokio_test::block_on(session.record_initial_history(InitialHistory::Resumed(
ResumedHistory {
conversation_id: ConversationId::default(),
history: rollout_items,
rollout_path: PathBuf::from("/tmp/resume.jsonl"),
},
)));
tokio_test::block_on(async {
session
.record_initial_history(InitialHistory::Resumed(ResumedHistory {
conversation_id: ConversationId::default(),
history: rollout_items,
rollout_path: PathBuf::from("/tmp/resume.jsonl"),
}))
.await
.expect("record resumed history");
});
let actual = tokio_test::block_on(async { session.state.lock().await.history_snapshot() });
assert_eq!(expected, actual);
@@ -2311,9 +2325,15 @@ mod tests {
#[test]
fn record_initial_history_reconstructs_forked_transcript() {
let (session, turn_context) = make_session_and_context();
let (rollout_items, expected) = sample_rollout(&session, &turn_context);
let (rollout_items, expected) =
sample_rollout(&session, &turn_context).expect("sample rollout");
tokio_test::block_on(session.record_initial_history(InitialHistory::Forked(rollout_items)));
tokio_test::block_on(async {
session
.record_initial_history(InitialHistory::Forked(rollout_items))
.await
.expect("record forked history");
});
let actual = tokio_test::block_on(async { session.state.lock().await.history_snapshot() });
assert_eq!(expected, actual);
@@ -2673,10 +2693,10 @@ mod tests {
_ctx: Arc<TurnContext>,
_input: Vec<UserInput>,
cancellation_token: CancellationToken,
) -> Option<String> {
) -> CodexResult<Option<String>> {
if self.listen_to_cancellation_token {
cancellation_token.cancelled().await;
return None;
return Ok(None);
}
loop {
sleep(Duration::from_secs(60)).await;
@@ -2685,7 +2705,7 @@ mod tests {
async fn abort(&self, session: Arc<SessionTaskContext>, ctx: Arc<TurnContext>) {
if let TaskKind::Review = self.kind {
exit_review_mode(session.clone_session(), ctx, None).await;
let _ = exit_review_mode(session.clone_session(), ctx, None).await;
}
}
}
@@ -2841,15 +2861,15 @@ mod tests {
fn sample_rollout(
session: &Session,
turn_context: &TurnContext,
) -> (Vec<RolloutItem>, Vec<ResponseItem>) {
) -> CodexResult<(Vec<RolloutItem>, Vec<ResponseItem>)> {
let mut rollout_items = Vec::new();
let mut live_history = ContextManager::new();
let mut live_history = ConversationHistory::new();
let initial_context = session.build_initial_context(turn_context);
for item in &initial_context {
rollout_items.push(RolloutItem::ResponseItem(item.clone()));
}
live_history.record_items(initial_context.iter());
live_history.record_items(initial_context.iter())?;
let user1 = ResponseItem::Message {
id: None,
@@ -2858,7 +2878,7 @@ mod tests {
text: "first user".to_string(),
}],
};
live_history.record_items(std::iter::once(&user1));
live_history.record_items(std::iter::once(&user1))?;
rollout_items.push(RolloutItem::ResponseItem(user1.clone()));
let assistant1 = ResponseItem::Message {
@@ -2868,7 +2888,7 @@ mod tests {
text: "assistant reply one".to_string(),
}],
};
live_history.record_items(std::iter::once(&assistant1));
live_history.record_items(std::iter::once(&assistant1))?;
rollout_items.push(RolloutItem::ResponseItem(assistant1.clone()));
let summary1 = "summary one";
@@ -2891,7 +2911,7 @@ mod tests {
text: "second user".to_string(),
}],
};
live_history.record_items(std::iter::once(&user2));
live_history.record_items(std::iter::once(&user2))?;
rollout_items.push(RolloutItem::ResponseItem(user2.clone()));
let assistant2 = ResponseItem::Message {
@@ -2901,7 +2921,7 @@ mod tests {
text: "assistant reply two".to_string(),
}],
};
live_history.record_items(std::iter::once(&assistant2));
live_history.record_items(std::iter::once(&assistant2))?;
rollout_items.push(RolloutItem::ResponseItem(assistant2.clone()));
let summary2 = "summary two";
@@ -2924,7 +2944,7 @@ mod tests {
text: "third user".to_string(),
}],
};
live_history.record_items(std::iter::once(&user3));
live_history.record_items(std::iter::once(&user3))?;
rollout_items.push(RolloutItem::ResponseItem(user3.clone()));
let assistant3 = ResponseItem::Message {
@@ -2934,10 +2954,10 @@ mod tests {
text: "assistant reply three".to_string(),
}],
};
live_history.record_items(std::iter::once(&assistant3));
live_history.record_items(std::iter::once(&assistant3))?;
rollout_items.push(RolloutItem::ResponseItem(assistant3.clone()));
(rollout_items, live_history.get_history())
Ok((rollout_items, live_history.get_history()))
}
#[tokio::test]
@@ -3091,7 +3111,7 @@ mod tests {
let display = mcp_init_error_display(server_name, Some(&entry), &err);
let expected = format!(
"GitHub MCP does not support OAuth. Log in by adding a personal access token (https://github.com/settings/personal-access-tokens) to your environment and config.toml:\n[mcp_servers.{server_name}]\nbearer_token_env_var = CODEX_GITHUB_PERSONAL_ACCESS_TOKEN"
"GitHub MCP does not support OAuth. Log in by adding `bearer_token_env_var = CODEX_GITHUB_PAT` in the `mcp_servers.{server_name}` section of your config.toml"
);
assert_eq!(expected, display);
@@ -3138,17 +3158,4 @@ mod tests {
assert_eq!(expected, display);
}
#[test]
fn mcp_init_error_display_includes_startup_timeout_hint() {
let server_name = "slow";
let err = anyhow::anyhow!("request timed out");
let display = mcp_init_error_display(server_name, None, &err);
assert_eq!(
"MCP client for `slow` timed out after 10 seconds. Add or adjust `startup_timeout_sec` in your config.toml:\n[mcp_servers.slow]\nstartup_timeout_sec = XX",
display
);
}
}

View File

@@ -39,35 +39,35 @@ struct HistoryBridgeTemplate<'a> {
pub(crate) async fn run_inline_auto_compact_task(
sess: Arc<Session>,
turn_context: Arc<TurnContext>,
) {
) -> CodexResult<()> {
let input = vec![UserInput::Text {
text: SUMMARIZATION_PROMPT.to_string(),
}];
run_compact_task_inner(sess, turn_context, input).await;
run_compact_task_inner(sess, turn_context, input).await
}
pub(crate) async fn run_compact_task(
sess: Arc<Session>,
turn_context: Arc<TurnContext>,
input: Vec<UserInput>,
) -> Option<String> {
) -> CodexResult<Option<String>> {
let start_event = EventMsg::TaskStarted(TaskStartedEvent {
model_context_window: turn_context.client.get_model_context_window(),
});
sess.send_event(&turn_context, start_event).await;
run_compact_task_inner(sess.clone(), turn_context, input).await;
None
run_compact_task_inner(sess.clone(), turn_context, input).await?;
Ok(None)
}
async fn run_compact_task_inner(
sess: Arc<Session>,
turn_context: Arc<TurnContext>,
input: Vec<UserInput>,
) {
) -> CodexResult<()> {
let initial_input_for_turn: ResponseInputItem = ResponseInputItem::from(input);
let mut history = sess.clone_history().await;
history.record_items(&[initial_input_for_turn.into()]);
history.record_items(&[initial_input_for_turn.into()])?;
let mut truncated_count = 0usize;
@@ -106,7 +106,7 @@ async fn run_compact_task_inner(
break;
}
Err(CodexErr::Interrupted) => {
return;
return Ok(());
}
Err(e @ CodexErr::ContextWindowExceeded) => {
if turn_input.len() > 1 {
@@ -124,7 +124,7 @@ async fn run_compact_task_inner(
message: e.to_string(),
});
sess.send_event(&turn_context, event).await;
return;
return Ok(());
}
Err(e) => {
if retries < max_retries {
@@ -142,7 +142,7 @@ async fn run_compact_task_inner(
message: e.to_string(),
});
sess.send_event(&turn_context, event).await;
return;
return Ok(());
}
}
}
@@ -164,6 +164,7 @@ async fn run_compact_task_inner(
message: "Compact task completed".to_string(),
});
sess.send_event(&turn_context, event).await;
Ok(())
}
pub fn content_items_to_text(content: &[ContentItem]) -> Option<String> {
@@ -252,7 +253,8 @@ async fn drain_to_completed(
};
match event {
Ok(ResponseEvent::OutputItemDone(item)) => {
sess.record_into_history(std::slice::from_ref(&item)).await;
sess.record_into_history(std::slice::from_ref(&item))
.await?;
}
Ok(ResponseEvent::RateLimits(snapshot)) => {
sess.update_rate_limits(turn_context, snapshot).await;

View File

@@ -1,3 +0,0 @@
mod manager;
pub(crate) use manager::ContextManager;
pub mod truncation;

View File

@@ -1,159 +0,0 @@
use codex_utils_string::take_bytes_at_char_boundary;
#[derive(Clone, Copy)]
pub(crate) struct TruncationConfig {
pub max_bytes: usize,
pub max_lines: usize,
pub truncation_notice: &'static str,
}
// Telemetry preview limits: keep log events smaller than model budgets.
pub(crate) const TELEMETRY_PREVIEW_MAX_BYTES: usize = 2 * 1024; // 2 KiB
pub(crate) const TELEMETRY_PREVIEW_MAX_LINES: usize = 64; // lines
pub(crate) const TELEMETRY_PREVIEW_TRUNCATION_NOTICE: &str =
"[... telemetry preview truncated ...]";
pub(crate) const CONTEXT_OUTPUT_TRUNCATION: TruncationConfig = TruncationConfig {
max_bytes: TELEMETRY_PREVIEW_MAX_BYTES,
max_lines: TELEMETRY_PREVIEW_MAX_LINES,
truncation_notice: TELEMETRY_PREVIEW_TRUNCATION_NOTICE,
};
pub(crate) fn truncate_with_config(content: &str, config: TruncationConfig) -> String {
let TruncationConfig {
max_bytes,
max_lines,
truncation_notice,
} = config;
let truncated_slice = take_bytes_at_char_boundary(content, max_bytes);
let truncated_by_bytes = truncated_slice.len() < content.len();
let mut preview = String::new();
let mut lines_iter = truncated_slice.lines();
for idx in 0..max_lines {
match lines_iter.next() {
Some(line) => {
if idx > 0 {
preview.push('\n');
}
preview.push_str(line);
}
None => break,
}
}
let truncated_by_lines = lines_iter.next().is_some();
if !truncated_by_bytes && !truncated_by_lines {
return content.to_string();
}
if preview.len() < truncated_slice.len()
&& truncated_slice
.as_bytes()
.get(preview.len())
.is_some_and(|byte| *byte == b'\n')
{
preview.push('\n');
}
if !preview.is_empty() && !preview.ends_with('\n') {
preview.push('\n');
}
preview.push_str(truncation_notice);
preview
}
pub(crate) fn truncate_context_output(content: &str) -> String {
truncate_with_config(content, CONTEXT_OUTPUT_TRUNCATION)
}
#[cfg(test)]
mod tests {
use super::*;
use pretty_assertions::assert_eq;
#[test]
fn truncate_with_config_returns_original_within_limits() {
let content = "short output";
let config = TruncationConfig {
max_bytes: 64,
max_lines: 5,
truncation_notice: "[notice]",
};
assert_eq!(truncate_with_config(content, config), content);
}
#[test]
fn truncate_with_config_truncates_by_bytes() {
let config = TruncationConfig {
max_bytes: 16,
max_lines: 10,
truncation_notice: "[notice]",
};
let content = "abcdefghijklmnopqrstuvwxyz";
let truncated = truncate_with_config(content, config);
assert!(truncated.contains("[notice]"));
}
#[test]
fn truncate_with_config_truncates_by_lines() {
let config = TruncationConfig {
max_bytes: 1024,
max_lines: 2,
truncation_notice: "[notice]",
};
let content = "l1\nl2\nl3\nl4";
let truncated = truncate_with_config(content, config);
assert!(truncated.lines().count() <= 3);
assert!(truncated.contains("[notice]"));
}
#[test]
fn telemetry_preview_returns_original_within_limits() {
let content = "short output";
let config = TruncationConfig {
max_bytes: TELEMETRY_PREVIEW_MAX_BYTES,
max_lines: TELEMETRY_PREVIEW_MAX_LINES,
truncation_notice: TELEMETRY_PREVIEW_TRUNCATION_NOTICE,
};
assert_eq!(truncate_with_config(content, config), content);
}
#[test]
fn telemetry_preview_truncates_by_bytes() {
let config = TruncationConfig {
max_bytes: TELEMETRY_PREVIEW_MAX_BYTES,
max_lines: TELEMETRY_PREVIEW_MAX_LINES,
truncation_notice: TELEMETRY_PREVIEW_TRUNCATION_NOTICE,
};
let content = "x".repeat(TELEMETRY_PREVIEW_MAX_BYTES + 8);
let preview = truncate_with_config(&content, config);
assert!(preview.contains(TELEMETRY_PREVIEW_TRUNCATION_NOTICE));
assert!(
preview.len()
<= TELEMETRY_PREVIEW_MAX_BYTES + TELEMETRY_PREVIEW_TRUNCATION_NOTICE.len() + 1
);
}
#[test]
fn telemetry_preview_truncates_by_lines() {
let config = TruncationConfig {
max_bytes: TELEMETRY_PREVIEW_MAX_BYTES,
max_lines: TELEMETRY_PREVIEW_MAX_LINES,
truncation_notice: TELEMETRY_PREVIEW_TRUNCATION_NOTICE,
};
let content = (0..(TELEMETRY_PREVIEW_MAX_LINES + 5))
.map(|idx| format!("line {idx}"))
.collect::<Vec<_>>()
.join("\n");
let preview = truncate_with_config(&content, config);
let lines: Vec<&str> = preview.lines().collect();
assert!(lines.len() <= TELEMETRY_PREVIEW_MAX_LINES + 1);
assert_eq!(lines.last(), Some(&TELEMETRY_PREVIEW_TRUNCATION_NOTICE));
}
}

View File

@@ -1,23 +1,31 @@
use crate::context_manager::truncation::truncate_context_output;
use std::sync::Arc;
use std::sync::OnceLock;
use codex_protocol::models::FunctionCallOutputPayload;
use codex_protocol::models::ResponseItem;
use codex_protocol::protocol::TokenUsage;
use codex_protocol::protocol::TokenUsageInfo;
use codex_utils_tokenizer::Tokenizer;
use tokio::task;
use tracing::error;
static TOKENIZER: OnceLock<Option<Arc<Tokenizer>>> = OnceLock::new();
use crate::error::CodexErr;
/// Transcript of conversation history
#[derive(Debug, Clone, Default)]
pub(crate) struct ContextManager {
#[derive(Debug, Clone)]
pub(crate) struct ConversationHistory {
/// The oldest items are at the beginning of the vector.
items: Vec<ResponseItem>,
token_info: Option<TokenUsageInfo>,
}
impl ContextManager {
impl ConversationHistory {
pub(crate) fn new() -> Self {
Self {
items: Vec::new(),
token_info: TokenUsageInfo::new_or_append(&None, &None, None),
token_info: None,
}
}
@@ -35,7 +43,7 @@ impl ContextManager {
}
/// `items` is ordered from oldest to newest.
pub(crate) fn record_items<I>(&mut self, items: I)
pub(crate) fn record_items<I>(&mut self, items: I) -> Result<(), CodexErr>
where
I: IntoIterator,
I::Item: std::ops::Deref<Target = ResponseItem>,
@@ -44,10 +52,10 @@ impl ContextManager {
if !is_api_message(&item) {
continue;
}
let processed = Self::process_item(&item);
self.items.push(processed);
self.validate_input(&item)?;
self.items.push(item.clone());
}
Ok(())
}
pub(crate) fn get_history(&mut self) -> Vec<ResponseItem> {
@@ -78,34 +86,70 @@ impl ContextManager {
self.remove_orphan_outputs();
}
fn process_item(item: &ResponseItem) -> ResponseItem {
match item {
ResponseItem::FunctionCallOutput { call_id, output } => {
let truncated_content = truncate_context_output(output.content.as_str());
ResponseItem::FunctionCallOutput {
call_id: call_id.clone(),
output: FunctionCallOutputPayload {
content: truncated_content,
success: output.success,
},
}
}
ResponseItem::CustomToolCallOutput { call_id, output } => {
let truncated = truncate_context_output(output);
ResponseItem::CustomToolCallOutput {
call_id: call_id.clone(),
output: truncated,
}
}
_ => item.clone(),
}
}
/// Returns a clone of the contents in the transcript.
fn contents(&self) -> Vec<ResponseItem> {
self.items.clone()
}
fn validate_input(&self, item: &ResponseItem) -> Result<(), CodexErr> {
match item {
ResponseItem::Message { content, .. } => {
self.validate_input_content_item(content)?;
Ok(())
}
ResponseItem::FunctionCall { .. }
| ResponseItem::FunctionCallOutput { .. }
| ResponseItem::CustomToolCall { .. }
| ResponseItem::CustomToolCallOutput { .. }
| ResponseItem::LocalShellCall { .. }
| ResponseItem::Reasoning { .. }
| ResponseItem::WebSearchCall { .. } => Ok(()),
ResponseItem::Other => Err(CodexErr::InvalidInput(format!("invalid input: {item:?}"))),
}
}
fn validate_input_content_item(
&self,
content: &[codex_protocol::models::ContentItem],
) -> Result<(), CodexErr> {
let Some(info) = &self.token_info else {
return Ok(());
};
// this will intentionally not check the context for the first turn before getting this information.
// it's acceptable tradeoff.
let Some(context_window) = info.model_context_window else {
return Ok(());
};
let tokenizer = match shared_tokenizer() {
Some(t) => t,
None => return Ok(()),
};
let mut input_tokens: i64 = 0;
for item in content {
match item {
codex_protocol::models::ContentItem::InputText { text } => {
input_tokens += tokenizer.count(text);
}
codex_protocol::models::ContentItem::InputImage { .. } => {
// no validation currently
}
codex_protocol::models::ContentItem::OutputText { .. } => {
// no validation currently
}
}
}
let prior_total = info.last_token_usage.total_tokens;
let combined_tokens = prior_total.saturating_add(input_tokens);
let threshold = context_window * 95 / 100;
if combined_tokens > threshold {
return Err(CodexErr::InvalidInput("input too large".to_string()));
}
Ok(())
}
fn ensure_call_outputs_present(&mut self) {
// Collect synthetic outputs to insert immediately after their calls.
// Store the insertion position (index of call) alongside the item so
@@ -131,7 +175,7 @@ impl ContextManager {
ResponseItem::FunctionCallOutput {
call_id: call_id.clone(),
output: FunctionCallOutputPayload {
content: truncate_context_output("aborted"),
content: "aborted".to_string(),
success: None,
},
},
@@ -154,7 +198,7 @@ impl ContextManager {
idx,
ResponseItem::CustomToolCallOutput {
call_id: call_id.clone(),
output: truncate_context_output("aborted"),
output: "aborted".to_string(),
},
));
}
@@ -178,7 +222,7 @@ impl ContextManager {
ResponseItem::FunctionCallOutput {
call_id: call_id.clone(),
output: FunctionCallOutputPayload {
content: truncate_context_output("aborted"),
content: "aborted".to_string(),
success: None,
},
},
@@ -274,10 +318,7 @@ impl ContextManager {
}
pub(crate) fn replace(&mut self, items: Vec<ResponseItem>) {
self.items = items
.into_iter()
.map(|item| Self::process_item(&item))
.collect();
self.items = items;
}
/// Removes the corresponding paired item for the provided `item`, if any.
@@ -371,6 +412,36 @@ fn error_or_panic(message: String) {
}
}
fn shared_tokenizer() -> Option<Arc<Tokenizer>> {
TOKENIZER.get().and_then(|opt| opt.as_ref().map(Arc::clone))
}
/// Kick off background initialization of the shared tokenizer without blocking the caller.
pub(crate) fn prefetch_tokenizer_in_background() {
if TOKENIZER.get().is_some() {
return;
}
// Spawn a background task to initialize the tokenizer. Use spawn_blocking in case
// initialization performs CPU-heavy work or file I/O.
tokio::spawn(async {
let result = task::spawn_blocking(Tokenizer::try_default).await;
match result {
Ok(Ok(tokenizer)) => {
let _ = TOKENIZER.set(Some(Arc::new(tokenizer)));
}
Ok(Err(error)) => {
error!("failed to create tokenizer: {error}");
let _ = TOKENIZER.set(None);
}
Err(join_error) => {
error!("failed to join tokenizer init task: {join_error}");
let _ = TOKENIZER.set(None);
}
}
});
}
/// Anything that is not a system message or "reasoning" message is considered
/// an API message.
fn is_api_message(message: &ResponseItem) -> bool {
@@ -390,8 +461,6 @@ fn is_api_message(message: &ResponseItem) -> bool {
#[cfg(test)]
mod tests {
use super::*;
use crate::context_manager::truncation::TELEMETRY_PREVIEW_MAX_BYTES;
use crate::context_manager::truncation::TELEMETRY_PREVIEW_TRUNCATION_NOTICE;
use codex_protocol::models::ContentItem;
use codex_protocol::models::FunctionCallOutputPayload;
use codex_protocol::models::LocalShellAction;
@@ -409,9 +478,9 @@ mod tests {
}
}
fn create_history_with_items(items: Vec<ResponseItem>) -> ContextManager {
let mut h = ContextManager::new();
h.record_items(items.iter());
fn create_history_with_items(items: Vec<ResponseItem>) -> ConversationHistory {
let mut h = ConversationHistory::new();
h.record_items(items.iter()).unwrap();
h
}
@@ -427,7 +496,7 @@ mod tests {
#[test]
fn filters_non_api_messages() {
let mut h = ContextManager::default();
let mut h = ConversationHistory::new();
// System message is not an API message; Other is ignored.
let system = ResponseItem::Message {
id: None,
@@ -436,12 +505,12 @@ mod tests {
text: "ignored".to_string(),
}],
};
h.record_items([&system, &ResponseItem::Other]);
h.record_items([&system, &ResponseItem::Other]).unwrap();
// User and assistant should be retained.
let u = user_msg("hi");
let a = assistant_msg("hello");
h.record_items([&u, &a]);
h.record_items([&u, &a]).unwrap();
let items = h.contents();
assert_eq!(
@@ -465,61 +534,6 @@ mod tests {
);
}
#[test]
fn record_items_truncates_function_call_output() {
let mut h = ContextManager::new();
let long_content = "a".repeat(TELEMETRY_PREVIEW_MAX_BYTES + 32);
let item = ResponseItem::FunctionCallOutput {
call_id: "call-long".to_string(),
output: FunctionCallOutputPayload {
content: long_content.clone(),
success: Some(true),
},
};
h.record_items([&item]);
let stored = h.contents();
let ResponseItem::FunctionCallOutput { output, .. } = &stored[0] else {
panic!("expected FunctionCallOutput variant");
};
assert!(
output
.content
.ends_with(TELEMETRY_PREVIEW_TRUNCATION_NOTICE),
"truncated content should end with notice"
);
assert!(
output.content.len() < long_content.len(),
"content should shrink after truncation"
);
}
#[test]
fn record_items_truncates_custom_tool_output() {
let mut h = ContextManager::new();
let long_content = "b".repeat(TELEMETRY_PREVIEW_MAX_BYTES + 64);
let item = ResponseItem::CustomToolCallOutput {
call_id: "custom-long".to_string(),
output: long_content.clone(),
};
h.record_items([&item]);
let stored = h.contents();
let ResponseItem::CustomToolCallOutput { output, .. } = &stored[0] else {
panic!("expected CustomToolCallOutput variant");
};
assert!(
output.ends_with(TELEMETRY_PREVIEW_TRUNCATION_NOTICE),
"truncated output should end with notice"
);
assert!(
output.len() < long_content.len(),
"output should shrink after truncation"
);
}
#[test]
fn remove_first_item_removes_matching_output_for_function_call() {
let items = vec![

View File

@@ -1,13 +1,5 @@
use crate::spawn::CODEX_SANDBOX_ENV_VAR;
use http::Error as HttpError;
use reqwest::IntoUrl;
use reqwest::Method;
use reqwest::Response;
use reqwest::header::HeaderName;
use reqwest::header::HeaderValue;
use serde::Serialize;
use std::collections::HashMap;
use std::fmt::Display;
use std::sync::LazyLock;
use std::sync::Mutex;
use std::sync::OnceLock;
@@ -30,130 +22,6 @@ use std::sync::OnceLock;
pub static USER_AGENT_SUFFIX: LazyLock<Mutex<Option<String>>> = LazyLock::new(|| Mutex::new(None));
pub const DEFAULT_ORIGINATOR: &str = "codex_cli_rs";
pub const CODEX_INTERNAL_ORIGINATOR_OVERRIDE_ENV_VAR: &str = "CODEX_INTERNAL_ORIGINATOR_OVERRIDE";
#[derive(Clone, Debug)]
pub struct CodexHttpClient {
inner: reqwest::Client,
}
impl CodexHttpClient {
fn new(inner: reqwest::Client) -> Self {
Self { inner }
}
pub fn get<U>(&self, url: U) -> CodexRequestBuilder
where
U: IntoUrl,
{
self.request(Method::GET, url)
}
pub fn post<U>(&self, url: U) -> CodexRequestBuilder
where
U: IntoUrl,
{
self.request(Method::POST, url)
}
pub fn request<U>(&self, method: Method, url: U) -> CodexRequestBuilder
where
U: IntoUrl,
{
let url_str = url.as_str().to_string();
CodexRequestBuilder::new(self.inner.request(method.clone(), url), method, url_str)
}
}
#[must_use = "requests are not sent unless `send` is awaited"]
#[derive(Debug)]
pub struct CodexRequestBuilder {
builder: reqwest::RequestBuilder,
method: Method,
url: String,
}
impl CodexRequestBuilder {
fn new(builder: reqwest::RequestBuilder, method: Method, url: String) -> Self {
Self {
builder,
method,
url,
}
}
fn map(self, f: impl FnOnce(reqwest::RequestBuilder) -> reqwest::RequestBuilder) -> Self {
Self {
builder: f(self.builder),
method: self.method,
url: self.url,
}
}
pub fn header<K, V>(self, key: K, value: V) -> Self
where
HeaderName: TryFrom<K>,
<HeaderName as TryFrom<K>>::Error: Into<HttpError>,
HeaderValue: TryFrom<V>,
<HeaderValue as TryFrom<V>>::Error: Into<HttpError>,
{
self.map(|builder| builder.header(key, value))
}
pub fn bearer_auth<T>(self, token: T) -> Self
where
T: Display,
{
self.map(|builder| builder.bearer_auth(token))
}
pub fn json<T>(self, value: &T) -> Self
where
T: ?Sized + Serialize,
{
self.map(|builder| builder.json(value))
}
pub async fn send(self) -> Result<Response, reqwest::Error> {
match self.builder.send().await {
Ok(response) => {
let request_ids = Self::extract_request_ids(&response);
tracing::debug!(
method = %self.method,
url = %self.url,
status = %response.status(),
request_ids = ?request_ids,
version = ?response.version(),
"Request completed"
);
Ok(response)
}
Err(error) => {
let status = error.status();
tracing::debug!(
method = %self.method,
url = %self.url,
status = status.map(|s| s.as_u16()),
error = %error,
"Request failed"
);
Err(error)
}
}
}
fn extract_request_ids(response: &Response) -> HashMap<String, String> {
["cf-ray", "x-request-id", "x-oai-request-id"]
.iter()
.filter_map(|&name| {
let header_name = HeaderName::from_static(name);
let value = response.headers().get(header_name)?;
let value = value.to_str().ok()?.to_owned();
Some((name.to_owned(), value))
})
.collect()
}
}
#[derive(Debug, Clone)]
pub struct Originator {
pub value: String,
@@ -256,8 +124,8 @@ fn sanitize_user_agent(candidate: String, fallback: &str) -> String {
}
}
/// Create an HTTP client with default `originator` and `User-Agent` headers set.
pub fn create_client() -> CodexHttpClient {
/// Create a reqwest client with default `originator` and `User-Agent` headers set.
pub fn create_client() -> reqwest::Client {
use reqwest::header::HeaderMap;
let mut headers = HeaderMap::new();
@@ -272,8 +140,7 @@ pub fn create_client() -> CodexHttpClient {
builder = builder.no_proxy();
}
let inner = builder.build().unwrap_or_else(|_| reqwest::Client::new());
CodexHttpClient::new(inner)
builder.build().unwrap_or_else(|_| reqwest::Client::new())
}
fn is_sandboxed() -> bool {

View File

@@ -158,6 +158,9 @@ pub enum CodexErr {
#[error("{0}")]
EnvVar(EnvVarError),
#[error("invalid input: {0}")]
InvalidInput(String),
}
impl From<CancelErr> for CodexErr {

View File

@@ -20,7 +20,7 @@ pub mod config_edit;
pub mod config_loader;
pub mod config_profile;
pub mod config_types;
mod context_manager;
mod conversation_history;
pub mod custom_prompts;
mod environment_context;
pub mod error;

View File

@@ -49,7 +49,7 @@ const MCP_TOOL_NAME_DELIMITER: &str = "__";
const MAX_TOOL_NAME_LENGTH: usize = 64;
/// Default timeout for initializing MCP server & initially listing tools.
pub const DEFAULT_STARTUP_TIMEOUT: Duration = Duration::from_secs(10);
const DEFAULT_STARTUP_TIMEOUT: Duration = Duration::from_secs(10);
/// Default timeout for individual tool calls.
const DEFAULT_TOOL_TIMEOUT: Duration = Duration::from_secs(60);

View File

@@ -6,8 +6,6 @@
//! key. These override or extend the defaults at runtime.
use crate::CodexAuth;
use crate::default_client::CodexHttpClient;
use crate::default_client::CodexRequestBuilder;
use codex_app_server_protocol::AuthMode;
use serde::Deserialize;
use serde::Serialize;
@@ -97,7 +95,7 @@ pub struct ModelProviderInfo {
impl ModelProviderInfo {
/// Construct a `POST` RequestBuilder for the given URL using the provided
/// [`CodexHttpClient`] applying:
/// reqwest Client applying:
/// • provider-specific headers (static + env based)
/// • Bearer auth header when an API key is available.
/// • Auth token for OAuth.
@@ -106,9 +104,9 @@ impl ModelProviderInfo {
/// one produced by [`ModelProviderInfo::api_key`].
pub async fn create_request_builder<'a>(
&'a self,
client: &'a CodexHttpClient,
client: &'a reqwest::Client,
auth: &Option<CodexAuth>,
) -> crate::error::Result<CodexRequestBuilder> {
) -> crate::error::Result<reqwest::RequestBuilder> {
let effective_auth = if let Some(secret_key) = &self.experimental_bearer_token {
Some(CodexAuth::from_api_key(secret_key))
} else {
@@ -189,9 +187,9 @@ impl ModelProviderInfo {
}
/// Apply provider-specific HTTP headers (both static and environment-based)
/// onto an existing [`CodexRequestBuilder`] and return the updated
/// onto an existing `reqwest::RequestBuilder` and return the updated
/// builder.
fn apply_http_headers(&self, mut builder: CodexRequestBuilder) -> CodexRequestBuilder {
fn apply_http_headers(&self, mut builder: reqwest::RequestBuilder) -> reqwest::RequestBuilder {
if let Some(extra) = &self.http_headers {
for (k, v) in extra {
builder = builder.header(k, v);

View File

@@ -1,5 +1,6 @@
use crate::codex::Session;
use crate::context_manager::ContextManager;
use crate::conversation_history::ConversationHistory;
use crate::error::Result as CodexResult;
use codex_protocol::models::FunctionCallOutputPayload;
use codex_protocol::models::ResponseInputItem;
use codex_protocol::models::ResponseItem;
@@ -11,9 +12,9 @@ use tracing::warn;
pub(crate) async fn process_items(
processed_items: Vec<crate::codex::ProcessedResponseItem>,
is_review_mode: bool,
review_thread_history: &mut ContextManager,
review_thread_history: &mut ConversationHistory,
sess: &Session,
) -> (Vec<ResponseInputItem>, Vec<ResponseItem>) {
) -> CodexResult<(Vec<ResponseInputItem>, Vec<ResponseItem>)> {
let mut items_to_record_in_conversation_history = Vec::<ResponseItem>::new();
let mut responses = Vec::<ResponseInputItem>::new();
for processed_response_item in processed_items {
@@ -102,11 +103,11 @@ pub(crate) async fn process_items(
// Only attempt to take the lock if there is something to record.
if !items_to_record_in_conversation_history.is_empty() {
if is_review_mode {
review_thread_history.record_items(items_to_record_in_conversation_history.iter());
review_thread_history.record_items(items_to_record_in_conversation_history.iter())?;
} else {
sess.record_conversation_items(&items_to_record_in_conversation_history)
.await;
.await?;
}
}
(responses, items_to_record_in_conversation_history)
Ok((responses, items_to_record_in_conversation_history))
}

View File

@@ -3,7 +3,8 @@
use codex_protocol::models::ResponseItem;
use crate::codex::SessionConfiguration;
use crate::context_manager::ContextManager;
use crate::conversation_history::ConversationHistory;
use crate::error::CodexErr;
use crate::protocol::RateLimitSnapshot;
use crate::protocol::TokenUsage;
use crate::protocol::TokenUsageInfo;
@@ -11,7 +12,7 @@ use crate::protocol::TokenUsageInfo;
/// Persistent, session-scoped state previously stored directly on `Session`.
pub(crate) struct SessionState {
pub(crate) session_configuration: SessionConfiguration,
pub(crate) history: ContextManager,
pub(crate) history: ConversationHistory,
pub(crate) latest_rate_limits: Option<RateLimitSnapshot>,
}
@@ -20,25 +21,26 @@ impl SessionState {
pub(crate) fn new(session_configuration: SessionConfiguration) -> Self {
Self {
session_configuration,
history: ContextManager::new(),
history: ConversationHistory::new(),
latest_rate_limits: None,
}
}
// History helpers
pub(crate) fn record_items<I>(&mut self, items: I)
pub(crate) fn record_items<I>(&mut self, items: I) -> Result<(), CodexErr>
where
I: IntoIterator,
I::Item: std::ops::Deref<Target = ResponseItem>,
{
self.history.record_items(items)
self.history.record_items(items)?;
Ok(())
}
pub(crate) fn history_snapshot(&mut self) -> Vec<ResponseItem> {
self.history.get_history()
}
pub(crate) fn clone_history(&self) -> ContextManager {
pub(crate) fn clone_history(&self) -> ConversationHistory {
self.history.clone()
}
@@ -66,7 +68,14 @@ impl SessionState {
pub(crate) fn token_info_and_rate_limits(
&self,
) -> (Option<TokenUsageInfo>, Option<RateLimitSnapshot>) {
(self.token_info(), self.latest_rate_limits.clone())
let info = self.token_info().and_then(|info| {
if info.total_token_usage.is_zero() && info.last_token_usage.is_zero() {
None
} else {
Some(info)
}
});
(info, self.latest_rate_limits.clone())
}
pub(crate) fn set_token_usage_full(&mut self, context_window: i64) {

View File

@@ -5,6 +5,7 @@ use tokio_util::sync::CancellationToken;
use crate::codex::TurnContext;
use crate::codex::compact;
use crate::error::Result as CodexResult;
use crate::state::TaskKind;
use codex_protocol::user_input::UserInput;
@@ -26,7 +27,7 @@ impl SessionTask for CompactTask {
ctx: Arc<TurnContext>,
input: Vec<UserInput>,
_cancellation_token: CancellationToken,
) -> Option<String> {
) -> CodexResult<Option<String>> {
compact::run_compact_task(session.clone_session(), ctx, input).await
}
}

View File

@@ -15,6 +15,8 @@ use tracing::warn;
use crate::codex::Session;
use crate::codex::TurnContext;
use crate::error::Result as CodexResult;
use crate::protocol::ErrorEvent;
use crate::protocol::EventMsg;
use crate::protocol::TaskCompleteEvent;
use crate::protocol::TurnAbortReason;
@@ -56,7 +58,7 @@ pub(crate) trait SessionTask: Send + Sync + 'static {
ctx: Arc<TurnContext>,
input: Vec<UserInput>,
cancellation_token: CancellationToken,
) -> Option<String>;
) -> CodexResult<Option<String>>;
async fn abort(&self, session: Arc<SessionTaskContext>, ctx: Arc<TurnContext>) {
let _ = (session, ctx);
@@ -86,7 +88,7 @@ impl Session {
let task_cancellation_token = cancellation_token.child_token();
tokio::spawn(async move {
let ctx_for_finish = Arc::clone(&ctx);
let last_agent_message = task_for_run
let run_result = task_for_run
.run(
Arc::clone(&session_ctx),
ctx,
@@ -98,8 +100,21 @@ impl Session {
if !task_cancellation_token.is_cancelled() {
// Emit completion uniformly from spawn site so all tasks share the same lifecycle.
let sess = session_ctx.clone_session();
sess.on_task_finished(ctx_for_finish, last_agent_message)
.await;
match run_result {
Ok(last_agent_message) => {
sess.on_task_finished(ctx_for_finish, last_agent_message)
.await;
}
Err(err) => {
let message = err.to_string();
sess.send_event(
ctx_for_finish.as_ref(),
EventMsg::Error(ErrorEvent { message }),
)
.await;
sess.on_task_finished(ctx_for_finish, None).await;
}
}
}
done_clone.notify_waiters();
})

View File

@@ -5,6 +5,7 @@ use tokio_util::sync::CancellationToken;
use crate::codex::TurnContext;
use crate::codex::run_task;
use crate::error::Result as CodexResult;
use crate::state::TaskKind;
use codex_protocol::user_input::UserInput;
@@ -26,7 +27,7 @@ impl SessionTask for RegularTask {
ctx: Arc<TurnContext>,
input: Vec<UserInput>,
cancellation_token: CancellationToken,
) -> Option<String> {
) -> CodexResult<Option<String>> {
let sess = session.clone_session();
run_task(sess, ctx, input, TaskKind::Regular, cancellation_token).await
}

View File

@@ -6,6 +6,7 @@ use tokio_util::sync::CancellationToken;
use crate::codex::TurnContext;
use crate::codex::exit_review_mode;
use crate::codex::run_task;
use crate::error::Result as CodexResult;
use crate::state::TaskKind;
use codex_protocol::user_input::UserInput;
@@ -27,12 +28,12 @@ impl SessionTask for ReviewTask {
ctx: Arc<TurnContext>,
input: Vec<UserInput>,
cancellation_token: CancellationToken,
) -> Option<String> {
) -> CodexResult<Option<String>> {
let sess = session.clone_session();
run_task(sess, ctx, input, TaskKind::Review, cancellation_token).await
}
async fn abort(&self, session: Arc<SessionTaskContext>, ctx: Arc<TurnContext>) {
exit_review_mode(session.clone_session(), ctx, None).await;
let _ = exit_review_mode(session.clone_session(), ctx, None).await;
}
}

View File

@@ -1,11 +1,15 @@
use crate::codex::Session;
use crate::codex::TurnContext;
use crate::tools::TELEMETRY_PREVIEW_MAX_BYTES;
use crate::tools::TELEMETRY_PREVIEW_MAX_LINES;
use crate::tools::TELEMETRY_PREVIEW_TRUNCATION_NOTICE;
use crate::turn_diff_tracker::TurnDiffTracker;
use codex_otel::otel_event_manager::OtelEventManager;
use codex_protocol::models::FunctionCallOutputPayload;
use codex_protocol::models::ResponseInputItem;
use codex_protocol::models::ShellToolCallParams;
use codex_protocol::protocol::FileChange;
use codex_utils_string::take_bytes_at_char_boundary;
use mcp_types::CallToolResult;
use std::borrow::Cow;
use std::collections::HashMap;
@@ -72,7 +76,7 @@ pub enum ToolOutput {
impl ToolOutput {
pub fn log_preview(&self) -> String {
match self {
ToolOutput::Function { content, .. } => content.clone(),
ToolOutput::Function { content, .. } => telemetry_preview(content),
ToolOutput::Mcp { result } => format!("{result:?}"),
}
}
@@ -107,6 +111,46 @@ impl ToolOutput {
}
}
fn telemetry_preview(content: &str) -> String {
let truncated_slice = take_bytes_at_char_boundary(content, TELEMETRY_PREVIEW_MAX_BYTES);
let truncated_by_bytes = truncated_slice.len() < content.len();
let mut preview = String::new();
let mut lines_iter = truncated_slice.lines();
for idx in 0..TELEMETRY_PREVIEW_MAX_LINES {
match lines_iter.next() {
Some(line) => {
if idx > 0 {
preview.push('\n');
}
preview.push_str(line);
}
None => break,
}
}
let truncated_by_lines = lines_iter.next().is_some();
if !truncated_by_bytes && !truncated_by_lines {
return content.to_string();
}
if preview.len() < truncated_slice.len()
&& truncated_slice
.as_bytes()
.get(preview.len())
.is_some_and(|byte| *byte == b'\n')
{
preview.push('\n');
}
if !preview.is_empty() && !preview.ends_with('\n') {
preview.push('\n');
}
preview.push_str(TELEMETRY_PREVIEW_TRUNCATION_NOTICE);
preview
}
#[cfg(test)]
mod tests {
use super::*;
@@ -152,6 +196,38 @@ mod tests {
other => panic!("expected FunctionCallOutput, got {other:?}"),
}
}
#[test]
fn telemetry_preview_returns_original_within_limits() {
let content = "short output";
assert_eq!(telemetry_preview(content), content);
}
#[test]
fn telemetry_preview_truncates_by_bytes() {
let content = "x".repeat(TELEMETRY_PREVIEW_MAX_BYTES + 8);
let preview = telemetry_preview(&content);
assert!(preview.contains(TELEMETRY_PREVIEW_TRUNCATION_NOTICE));
assert!(
preview.len()
<= TELEMETRY_PREVIEW_MAX_BYTES + TELEMETRY_PREVIEW_TRUNCATION_NOTICE.len() + 1
);
}
#[test]
fn telemetry_preview_truncates_by_lines() {
let content = (0..(TELEMETRY_PREVIEW_MAX_LINES + 5))
.map(|idx| format!("line {idx}"))
.collect::<Vec<_>>()
.join("\n");
let preview = telemetry_preview(&content);
let lines: Vec<&str> = preview.lines().collect();
assert!(lines.len() <= TELEMETRY_PREVIEW_MAX_LINES + 1);
assert_eq!(lines.last(), Some(&TELEMETRY_PREVIEW_TRUNCATION_NOTICE));
}
}
#[derive(Clone, Debug)]

View File

@@ -22,6 +22,12 @@ pub(crate) const MODEL_FORMAT_HEAD_LINES: usize = MODEL_FORMAT_MAX_LINES / 2;
pub(crate) const MODEL_FORMAT_TAIL_LINES: usize = MODEL_FORMAT_MAX_LINES - MODEL_FORMAT_HEAD_LINES; // 128
pub(crate) const MODEL_FORMAT_HEAD_BYTES: usize = MODEL_FORMAT_MAX_BYTES / 2;
// Telemetry preview limits: keep log events smaller than model budgets.
pub(crate) const TELEMETRY_PREVIEW_MAX_BYTES: usize = 2 * 1024; // 2 KiB
pub(crate) const TELEMETRY_PREVIEW_MAX_LINES: usize = 64; // lines
pub(crate) const TELEMETRY_PREVIEW_TRUNCATION_NOTICE: &str =
"[... telemetry preview truncated ...]";
/// Format the combined exec output for sending back to the model.
/// Includes exit code and duration metadata; truncates large bodies safely.
pub fn format_exec_output_for_model(exec_output: &ExecToolCallOutput) -> String {

View File

@@ -279,11 +279,6 @@ async fn auto_compact_runs_after_token_limit_hit() {
ev_completed_with_tokens("r2", 330_000),
]);
let sse3 = sse(vec![
ev_assistant_message("m3", AUTO_SUMMARY_TEXT),
ev_completed_with_tokens("r3", 200),
]);
let first_matcher = |req: &wiremock::Request| {
let body = std::str::from_utf8(&req.body).unwrap_or("");
body.contains(FIRST_AUTO_MSG)
@@ -300,12 +295,6 @@ async fn auto_compact_runs_after_token_limit_hit() {
};
mount_sse_once_match(&server, second_matcher, sse2).await;
let third_matcher = |req: &wiremock::Request| {
let body = std::str::from_utf8(&req.body).unwrap_or("");
body.contains("You have exceeded the maximum number of tokens")
};
mount_sse_once_match(&server, third_matcher, sse3).await;
let model_provider = ModelProviderInfo {
base_url: Some(format!("{}/v1", server.uri())),
..built_in_model_providers()["openai"].clone()
@@ -342,69 +331,28 @@ async fn auto_compact_runs_after_token_limit_hit() {
.await
.unwrap();
let error_event = wait_for_event(&codex, |ev| matches!(ev, EventMsg::Error(_))).await;
let EventMsg::Error(error_event) = error_event else {
unreachable!("wait_for_event returned unexpected payload");
};
assert_eq!(error_event.message, "invalid input: input too large");
wait_for_event(&codex, |ev| matches!(ev, EventMsg::TaskComplete(_))).await;
// wait_for_event(&codex, |ev| matches!(ev, EventMsg::TaskComplete(_))).await;
let requests = server.received_requests().await.unwrap();
assert!(
requests.len() >= 3,
"auto compact should add at least a third request, got {}",
requests.len()
assert_eq!(
requests.len(),
2,
"auto compact should reject oversize prompts before issuing another request"
);
let is_auto_compact = |req: &wiremock::Request| {
let saw_compact_prompt = requests.iter().any(|req| {
std::str::from_utf8(&req.body)
.unwrap_or("")
.contains("You have exceeded the maximum number of tokens")
};
let auto_compact_count = requests.iter().filter(|req| is_auto_compact(req)).count();
assert_eq!(
auto_compact_count, 1,
"expected exactly one auto compact request"
);
let auto_compact_index = requests
.iter()
.enumerate()
.find_map(|(idx, req)| is_auto_compact(req).then_some(idx))
.expect("auto compact request missing");
assert_eq!(
auto_compact_index, 2,
"auto compact should add a third request"
);
let body_first = requests[0].body_json::<serde_json::Value>().unwrap();
let body3 = requests[auto_compact_index]
.body_json::<serde_json::Value>()
.unwrap();
let instructions = body3
.get("instructions")
.and_then(|v| v.as_str())
.unwrap_or_default();
let baseline_instructions = body_first
.get("instructions")
.and_then(|v| v.as_str())
.unwrap_or_default()
.to_string();
assert_eq!(
instructions, baseline_instructions,
"auto compact should keep the standard developer instructions",
);
let input3 = body3.get("input").and_then(|v| v.as_array()).unwrap();
let last3 = input3
.last()
.expect("auto compact request should append a user message");
assert_eq!(last3.get("type").and_then(|v| v.as_str()), Some("message"));
assert_eq!(last3.get("role").and_then(|v| v.as_str()), Some("user"));
let last_text = last3
.get("content")
.and_then(|v| v.as_array())
.and_then(|items| items.first())
.and_then(|item| item.get("text"))
.and_then(|text| text.as_str())
.unwrap_or_default();
assert_eq!(
last_text, SUMMARIZATION_PROMPT,
"auto compact should send the summarization prompt as a user message",
});
assert!(
!saw_compact_prompt,
"no auto compact request should be sent when the summarization prompt exceeds the limit"
);
}
@@ -869,7 +817,7 @@ async fn auto_compact_triggers_after_function_call_over_95_percent_usage() {
let server = start_mock_server().await;
let context_window = 100;
let context_window = 20_000;
let limit = context_window * 90 / 100;
let over_limit_tokens = context_window * 95 / 100 + 1;

View File

@@ -0,0 +1,69 @@
use codex_core::protocol::EventMsg;
use codex_core::protocol::Op;
use codex_protocol::user_input::UserInput;
use core_test_support::responses;
use core_test_support::responses::ev_assistant_message;
use core_test_support::responses::ev_completed;
use core_test_support::responses::ev_response_created;
use core_test_support::responses::sse;
use core_test_support::responses::start_mock_server;
use core_test_support::test_codex::test_codex;
use core_test_support::wait_for_event_with_timeout;
use std::sync::Arc;
use std::time::Duration;
use wiremock::matchers::any;
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn input_validation_should_fail_for_too_large_input() {
let server = start_mock_server().await;
let fixture = test_codex().build(&server).await.unwrap();
let codex = Arc::clone(&fixture.codex);
// First: normal message with a mocked assistant response
let first_response = sse(vec![
ev_response_created("resp-1"),
ev_assistant_message("msg-1", "ok"),
ev_completed("resp-1"),
]);
responses::mount_sse_once_match(&server, any(), first_response).await;
codex
.submit(Op::UserInput {
items: vec![UserInput::Text {
text: "hello world".into(),
}],
})
.await
.unwrap();
// Wait for the normal turn to complete before sending the oversized input
let turn_timeout = Duration::from_secs(1);
wait_for_event_with_timeout(
&codex,
|ev| matches!(ev, EventMsg::TaskComplete(_)),
turn_timeout,
)
.await;
// Then: 300k-token message should trigger validation error
let wait_timeout = Duration::from_millis(100);
let input_300_tokens = "token ".repeat(300_000);
codex
.submit(Op::UserInput {
items: vec![UserInput::Text {
text: input_300_tokens,
}],
})
.await
.unwrap();
let error_event =
wait_for_event_with_timeout(&codex, |ev| matches!(ev, EventMsg::Error(_)), wait_timeout)
.await;
let EventMsg::Error(error_event) = error_event else {
unreachable!("wait_for_event_with_timeout returned unexpected payload");
};
assert_eq!(error_event.message, "invalid input: input too large");
}

View File

@@ -13,6 +13,7 @@ mod compact_resume_fork;
mod exec;
mod fork_conversation;
mod grep_files;
mod input_validation;
mod items;
mod json_result;
mod list_dir;

View File

@@ -156,12 +156,6 @@ async fn unified_exec_emits_exec_command_end_event() -> Result<()> {
"cmd": "/bin/echo END-EVENT".to_string(),
"yield_time_ms": 250,
});
let poll_call_id = "uexec-end-event-poll";
let poll_args = json!({
"chars": "",
"session_id": 0,
"yield_time_ms": 250,
});
let responses = vec![
sse(vec![
@@ -171,17 +165,8 @@ async fn unified_exec_emits_exec_command_end_event() -> Result<()> {
]),
sse(vec![
ev_response_created("resp-2"),
ev_function_call(
poll_call_id,
"write_stdin",
&serde_json::to_string(&poll_args)?,
),
ev_completed("resp-2"),
]),
sse(vec![
ev_response_created("resp-3"),
ev_assistant_message("msg-1", "finished"),
ev_completed("resp-3"),
ev_completed("resp-2"),
]),
];
mount_sse_sequence(&server, responses).await;

View File

@@ -12,7 +12,7 @@ use anyhow::anyhow;
use codex_protocol::ConversationId;
use tracing_subscriber::fmt::writer::MakeWriter;
const DEFAULT_MAX_BYTES: usize = 4 * 1024 * 1024; // 4 MiB
const DEFAULT_MAX_BYTES: usize = 2 * 1024 * 1024; // 2 MiB
const SENTRY_DSN: &str =
"https://ae32ed50620d7a7792c1ce5df38b3e3e@o33249.ingest.us.sentry.io/4510195390611458";
const UPLOAD_TIMEOUT_SECS: u64 = 10;

View File

@@ -575,9 +575,11 @@ pub struct TokenUsage {
pub total_tokens: i64,
}
#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, TS)]
#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, TS, Default)]
pub struct TokenUsageInfo {
/// The total token usage for the session. accumulated from all turns.
pub total_token_usage: TokenUsage,
/// The token usage for the last turn. Received from the API. It's total tokens is the whole window size.
pub last_token_usage: TokenUsage,
#[ts(type = "number | null")]
pub model_context_window: Option<i64>,

View File

@@ -1631,7 +1631,6 @@ impl ChatWidget {
context_usage,
&self.conversation_id,
self.rate_limit_snapshot.as_ref(),
Local::now(),
));
}

View File

@@ -3,8 +3,6 @@ use crate::history_cell::HistoryCell;
use crate::history_cell::PlainHistoryCell;
use crate::history_cell::with_border_with_inner_width;
use crate::version::CODEX_CLI_VERSION;
use chrono::DateTime;
use chrono::Local;
use codex_common::create_config_summary_entries;
use codex_core::config::Config;
use codex_core::protocol::SandboxPolicy;
@@ -27,7 +25,6 @@ use super::helpers::format_directory_display;
use super::helpers::format_tokens_compact;
use super::rate_limits::RateLimitSnapshotDisplay;
use super::rate_limits::StatusRateLimitData;
use super::rate_limits::StatusRateLimitRow;
use super::rate_limits::compose_rate_limit_data;
use super::rate_limits::format_status_limit_summary;
use super::rate_limits::render_status_limit_progress_bar;
@@ -67,17 +64,9 @@ pub(crate) fn new_status_output(
context_usage: Option<&TokenUsage>,
session_id: &Option<ConversationId>,
rate_limits: Option<&RateLimitSnapshotDisplay>,
now: DateTime<Local>,
) -> CompositeHistoryCell {
let command = PlainHistoryCell::new(vec!["/status".magenta().into()]);
let card = StatusHistoryCell::new(
config,
total_usage,
context_usage,
session_id,
rate_limits,
now,
);
let card = StatusHistoryCell::new(config, total_usage, context_usage, session_id, rate_limits);
CompositeHistoryCell::new(vec![Box::new(command), Box::new(card)])
}
@@ -89,7 +78,6 @@ impl StatusHistoryCell {
context_usage: Option<&TokenUsage>,
session_id: &Option<ConversationId>,
rate_limits: Option<&RateLimitSnapshotDisplay>,
now: DateTime<Local>,
) -> Self {
let config_entries = create_config_summary_entries(config);
let (model_name, model_details) = compose_model_display(config, &config_entries);
@@ -120,7 +108,7 @@ impl StatusHistoryCell {
output: total_usage.output_tokens,
context_window,
};
let rate_limits = compose_rate_limit_data(rate_limits, now);
let rate_limits = compose_rate_limit_data(rate_limits);
Self {
model_name,
@@ -183,66 +171,47 @@ impl StatusHistoryCell {
];
}
self.rate_limit_row_lines(rows_data, available_inner_width, formatter)
}
StatusRateLimitData::Stale(rows_data) => {
let mut lines =
self.rate_limit_row_lines(rows_data, available_inner_width, formatter);
lines.push(formatter.line(
"Warning",
vec![Span::from("limits may be stale - start new turn to refresh.").dim()],
));
let mut lines = Vec::with_capacity(rows_data.len() * 2);
for row in rows_data {
let value_spans = vec![
Span::from(render_status_limit_progress_bar(row.percent_used)),
Span::from(" "),
Span::from(format_status_limit_summary(row.percent_used)),
];
let base_spans = formatter.full_spans(row.label.as_str(), value_spans);
let base_line = Line::from(base_spans.clone());
if let Some(resets_at) = row.resets_at.as_ref() {
let resets_span = Span::from(format!("(resets {resets_at})")).dim();
let mut inline_spans = base_spans.clone();
inline_spans.push(Span::from(" ").dim());
inline_spans.push(resets_span.clone());
if line_display_width(&Line::from(inline_spans.clone()))
<= available_inner_width
{
lines.push(Line::from(inline_spans));
} else {
lines.push(base_line);
lines.push(formatter.continuation(vec![resets_span]));
}
} else {
lines.push(base_line);
}
}
lines
}
StatusRateLimitData::Missing => {
vec![formatter.line(
"Limits",
vec![
Span::from("visit ").dim(),
"chatgpt.com/codex/settings/usage".cyan().underlined(),
],
vec![Span::from("send a message to load usage data").dim()],
)]
}
}
}
fn rate_limit_row_lines(
&self,
rows: &[StatusRateLimitRow],
available_inner_width: usize,
formatter: &FieldFormatter,
) -> Vec<Line<'static>> {
let mut lines = Vec::with_capacity(rows.len().saturating_mul(2));
for row in rows {
let value_spans = vec![
Span::from(render_status_limit_progress_bar(row.percent_used)),
Span::from(" "),
Span::from(format_status_limit_summary(row.percent_used)),
];
let base_spans = formatter.full_spans(row.label.as_str(), value_spans);
let base_line = Line::from(base_spans.clone());
if let Some(resets_at) = row.resets_at.as_ref() {
let resets_span = Span::from(format!("(resets {resets_at})")).dim();
let mut inline_spans = base_spans.clone();
inline_spans.push(Span::from(" ").dim());
inline_spans.push(resets_span.clone());
if line_display_width(&Line::from(inline_spans.clone())) <= available_inner_width {
lines.push(Line::from(inline_spans));
} else {
lines.push(base_line);
lines.push(formatter.continuation(vec![resets_span]));
}
} else {
lines.push(base_line);
}
}
lines
}
fn collect_rate_limit_labels(&self, seen: &mut BTreeSet<String>, labels: &mut Vec<String>) {
match &self.rate_limits {
StatusRateLimitData::Available(rows) => {
@@ -254,12 +223,6 @@ impl StatusHistoryCell {
}
}
}
StatusRateLimitData::Stale(rows) => {
for row in rows {
push_label(labels, seen, row.label.as_str());
}
push_label(labels, seen, "Warning");
}
StatusRateLimitData::Missing => push_label(labels, seen, "Limits"),
}
}

View File

@@ -2,7 +2,6 @@ use crate::chatwidget::get_limits_duration;
use super::helpers::format_reset_timestamp;
use chrono::DateTime;
use chrono::Duration as ChronoDuration;
use chrono::Local;
use chrono::Utc;
use codex_core::protocol::RateLimitSnapshot;
@@ -22,12 +21,9 @@ pub(crate) struct StatusRateLimitRow {
#[derive(Debug, Clone)]
pub(crate) enum StatusRateLimitData {
Available(Vec<StatusRateLimitRow>),
Stale(Vec<StatusRateLimitRow>),
Missing,
}
pub(crate) const RATE_LIMIT_STALE_THRESHOLD_MINUTES: i64 = 15;
#[derive(Debug, Clone)]
pub(crate) struct RateLimitWindowDisplay {
pub used_percent: f64,
@@ -53,7 +49,6 @@ impl RateLimitWindowDisplay {
#[derive(Debug, Clone)]
pub(crate) struct RateLimitSnapshotDisplay {
pub captured_at: DateTime<Local>,
pub primary: Option<RateLimitWindowDisplay>,
pub secondary: Option<RateLimitWindowDisplay>,
}
@@ -63,7 +58,6 @@ pub(crate) fn rate_limit_snapshot_display(
captured_at: DateTime<Local>,
) -> RateLimitSnapshotDisplay {
RateLimitSnapshotDisplay {
captured_at,
primary: snapshot
.primary
.as_ref()
@@ -77,7 +71,6 @@ pub(crate) fn rate_limit_snapshot_display(
pub(crate) fn compose_rate_limit_data(
snapshot: Option<&RateLimitSnapshotDisplay>,
now: DateTime<Local>,
) -> StatusRateLimitData {
match snapshot {
Some(snapshot) => {
@@ -109,13 +102,8 @@ pub(crate) fn compose_rate_limit_data(
});
}
let is_stale = now.signed_duration_since(snapshot.captured_at)
> ChronoDuration::minutes(RATE_LIMIT_STALE_THRESHOLD_MINUTES);
if rows.is_empty() {
StatusRateLimitData::Available(vec![])
} else if is_stale {
StatusRateLimitData::Stale(rows)
} else {
StatusRateLimitData::Available(rows)
}

View File

@@ -15,5 +15,5 @@ expression: sanitized
│ │
│ Token usage: 750 total (500 input + 250 output) │
│ Context window: 100% left (750 used / 272K) │
│ Limits: visit chatgpt.com/codex/settings/usage
│ Limits: send a message to load usage data
╰─────────────────────────────────────────────────────────────────╯

View File

@@ -1,21 +0,0 @@
---
source: tui/src/status/tests.rs
expression: sanitized
---
/status
╭─────────────────────────────────────────────────────────────────────╮
│ >_ OpenAI Codex (v0.0.0) │
│ │
│ Model: gpt-5-codex (reasoning none, summaries auto) │
│ Directory: [[workspace]] │
│ Approval: on-request │
│ Sandbox: read-only │
│ Agents.md: <none> │
│ │
│ Token usage: 1.9K total (1K input + 900 output) │
│ Context window: 100% left (2.1K used / 272K) │
│ 5h limit: [███████████████░░░░░] 72% used (resets 03:14) │
│ Weekly limit: [████████░░░░░░░░░░░░] 40% used (resets 03:34) │
│ Warning: limits may be stale - start new turn to refresh. │
╰─────────────────────────────────────────────────────────────────────╯

View File

@@ -111,14 +111,7 @@ fn status_snapshot_includes_reasoning_details() {
};
let rate_display = rate_limit_snapshot_display(&snapshot, captured_at);
let composite = new_status_output(
&config,
&usage,
Some(&usage),
&None,
Some(&rate_display),
captured_at,
);
let composite = new_status_output(&config, &usage, Some(&usage), &None, Some(&rate_display));
let mut rendered_lines = render_lines(&composite.display_lines(80));
if cfg!(windows) {
for line in &mut rendered_lines {
@@ -159,14 +152,7 @@ fn status_snapshot_includes_monthly_limit() {
};
let rate_display = rate_limit_snapshot_display(&snapshot, captured_at);
let composite = new_status_output(
&config,
&usage,
Some(&usage),
&None,
Some(&rate_display),
captured_at,
);
let composite = new_status_output(&config, &usage, Some(&usage), &None, Some(&rate_display));
let mut rendered_lines = render_lines(&composite.display_lines(80));
if cfg!(windows) {
for line in &mut rendered_lines {
@@ -192,12 +178,7 @@ fn status_card_token_usage_excludes_cached_tokens() {
total_tokens: 2_100,
};
let now = chrono::Local
.with_ymd_and_hms(2024, 1, 1, 0, 0, 0)
.single()
.expect("timestamp");
let composite = new_status_output(&config, &usage, Some(&usage), &None, None, now);
let composite = new_status_output(&config, &usage, Some(&usage), &None, None);
let rendered = render_lines(&composite.display_lines(120));
assert!(
@@ -238,14 +219,7 @@ fn status_snapshot_truncates_in_narrow_terminal() {
};
let rate_display = rate_limit_snapshot_display(&snapshot, captured_at);
let composite = new_status_output(
&config,
&usage,
Some(&usage),
&None,
Some(&rate_display),
captured_at,
);
let composite = new_status_output(&config, &usage, Some(&usage), &None, Some(&rate_display));
let mut rendered_lines = render_lines(&composite.display_lines(46));
if cfg!(windows) {
for line in &mut rendered_lines {
@@ -272,12 +246,7 @@ fn status_snapshot_shows_missing_limits_message() {
total_tokens: 750,
};
let now = chrono::Local
.with_ymd_and_hms(2024, 2, 3, 4, 5, 6)
.single()
.expect("timestamp");
let composite = new_status_output(&config, &usage, Some(&usage), &None, None, now);
let composite = new_status_output(&config, &usage, Some(&usage), &None, None);
let mut rendered_lines = render_lines(&composite.display_lines(80));
if cfg!(windows) {
for line in &mut rendered_lines {
@@ -313,66 +282,7 @@ fn status_snapshot_shows_empty_limits_message() {
.expect("timestamp");
let rate_display = rate_limit_snapshot_display(&snapshot, captured_at);
let composite = new_status_output(
&config,
&usage,
Some(&usage),
&None,
Some(&rate_display),
captured_at,
);
let mut rendered_lines = render_lines(&composite.display_lines(80));
if cfg!(windows) {
for line in &mut rendered_lines {
*line = line.replace('\\', "/");
}
}
let sanitized = sanitize_directory(rendered_lines).join("\n");
assert_snapshot!(sanitized);
}
#[test]
fn status_snapshot_shows_stale_limits_message() {
let temp_home = TempDir::new().expect("temp home");
let mut config = test_config(&temp_home);
config.model = "gpt-5-codex".to_string();
config.cwd = PathBuf::from("/workspace/tests");
let usage = TokenUsage {
input_tokens: 1_200,
cached_input_tokens: 200,
output_tokens: 900,
reasoning_output_tokens: 150,
total_tokens: 2_250,
};
let captured_at = chrono::Local
.with_ymd_and_hms(2024, 1, 2, 3, 4, 5)
.single()
.expect("timestamp");
let snapshot = RateLimitSnapshot {
primary: Some(RateLimitWindow {
used_percent: 72.5,
window_minutes: Some(300),
resets_at: Some(reset_at_from(&captured_at, 600)),
}),
secondary: Some(RateLimitWindow {
used_percent: 40.0,
window_minutes: Some(10_080),
resets_at: Some(reset_at_from(&captured_at, 1_800)),
}),
};
let rate_display = rate_limit_snapshot_display(&snapshot, captured_at);
let now = captured_at + ChronoDuration::minutes(20);
let composite = new_status_output(
&config,
&usage,
Some(&usage),
&None,
Some(&rate_display),
now,
);
let composite = new_status_output(&config, &usage, Some(&usage), &None, Some(&rate_display));
let mut rendered_lines = render_lines(&composite.display_lines(80));
if cfg!(windows) {
for line in &mut rendered_lines {
@@ -404,12 +314,7 @@ fn status_context_window_uses_last_usage() {
total_tokens: 13_679,
};
let now = chrono::Local
.with_ymd_and_hms(2024, 6, 1, 12, 0, 0)
.single()
.expect("timestamp");
let composite = new_status_output(&config, &total_usage, Some(&last_usage), &None, None, now);
let composite = new_status_output(&config, &total_usage, Some(&last_usage), &None, None);
let rendered_lines = render_lines(&composite.display_lines(80));
let context_line = rendered_lines
.into_iter()

View File

@@ -107,6 +107,12 @@ impl Tokenizer {
}
}
impl fmt::Debug for Tokenizer {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("Tokenizer").finish()
}
}
#[cfg(test)]
mod tests {
use super::*;

View File

@@ -42,14 +42,3 @@ Running Codex directly on Windows may work, but is not officially supported. We
### Where should I start after installation?
Follow the quick setup in [Install & build](./install.md) and then jump into [Getting started](./getting-started.md) for interactive usage tips, prompt examples, and AGENTS.md guidance.
### `brew upgrade codex` isn't upgrading me
If you're running Codex v0.46.0 or older, `brew upgrade codex` will not move you to the latest version because we migrated from a Homebrew formula to a cask. To upgrade, uninstall the existing oudated formula and then install the new cask:
```bash
brew uninstall --formula codex
brew install --cask codex
```
After reinstalling, `brew upgrade --cask codex` will keep future releases up to date.