mirror of
https://github.com/openai/codex.git
synced 2026-02-07 09:23:47 +00:00
Compare commits
15 Commits
centralize
...
input-vali
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
257d5eceff | ||
|
|
56dfa60801 | ||
|
|
50055c7ed5 | ||
|
|
89e9bb1d42 | ||
|
|
81d0003f0f | ||
|
|
5d7983520b | ||
|
|
4e7089a8ab | ||
|
|
697367cd3f | ||
|
|
6ffbd0d4e3 | ||
|
|
79c628a823 | ||
|
|
9446de0923 | ||
|
|
0f02954edb | ||
|
|
7a8da22d7e | ||
|
|
8f8ca17da0 | ||
|
|
8b095d3cf1 |
@@ -33,8 +33,6 @@ Then simply run `codex` to get started:
|
||||
codex
|
||||
```
|
||||
|
||||
If you're running into upgrade issues with Homebrew, see the [FAQ entry on brew upgrade codex](./docs/faq.md#brew-update-codex-isnt-upgrading-me).
|
||||
|
||||
<details>
|
||||
<summary>You can also go to the <a href="https://github.com/openai/codex/releases/latest">latest GitHub Release</a> and download the appropriate binary for your platform.</summary>
|
||||
|
||||
|
||||
1
codex-rs/Cargo.lock
generated
1
codex-rs/Cargo.lock
generated
@@ -1075,7 +1075,6 @@ dependencies = [
|
||||
"escargot",
|
||||
"eventsource-stream",
|
||||
"futures",
|
||||
"http",
|
||||
"indexmap 2.10.0",
|
||||
"landlock",
|
||||
"libc",
|
||||
|
||||
@@ -116,7 +116,6 @@ env_logger = "0.11.5"
|
||||
escargot = "0.5"
|
||||
eventsource-stream = "0.2.3"
|
||||
futures = { version = "0.3", default-features = false }
|
||||
http = "1.3.1"
|
||||
icu_decimal = "2.0.0"
|
||||
icu_locale_core = "2.0.0"
|
||||
ignore = "0.4.23"
|
||||
|
||||
@@ -23,7 +23,6 @@ use std::io::Write;
|
||||
use std::path::Path;
|
||||
use std::path::PathBuf;
|
||||
use std::process::Command;
|
||||
use ts_rs::ExportError;
|
||||
use ts_rs::TS;
|
||||
|
||||
const HEADER: &str = "// GENERATED CODE! DO NOT MODIFY BY HAND!\n\n";
|
||||
@@ -105,19 +104,6 @@ macro_rules! for_each_schema_type {
|
||||
};
|
||||
}
|
||||
|
||||
fn export_ts_with_context<F>(label: &str, export: F) -> Result<()>
|
||||
where
|
||||
F: FnOnce() -> std::result::Result<(), ExportError>,
|
||||
{
|
||||
match export() {
|
||||
Ok(()) => Ok(()),
|
||||
Err(ExportError::CannotBeExported(ty)) => Err(anyhow!(
|
||||
"failed to export {label}: dependency {ty} cannot be exported"
|
||||
)),
|
||||
Err(err) => Err(err.into()),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn generate_types(out_dir: &Path, prettier: Option<&Path>) -> Result<()> {
|
||||
generate_ts(out_dir, prettier)?;
|
||||
generate_json(out_dir)?;
|
||||
@@ -127,17 +113,13 @@ pub fn generate_types(out_dir: &Path, prettier: Option<&Path>) -> Result<()> {
|
||||
pub fn generate_ts(out_dir: &Path, prettier: Option<&Path>) -> Result<()> {
|
||||
ensure_dir(out_dir)?;
|
||||
|
||||
export_ts_with_context("ClientRequest", || ClientRequest::export_all_to(out_dir))?;
|
||||
export_ts_with_context("client responses", || export_client_responses(out_dir))?;
|
||||
export_ts_with_context("ClientNotification", || {
|
||||
ClientNotification::export_all_to(out_dir)
|
||||
})?;
|
||||
ClientRequest::export_all_to(out_dir)?;
|
||||
export_client_responses(out_dir)?;
|
||||
ClientNotification::export_all_to(out_dir)?;
|
||||
|
||||
export_ts_with_context("ServerRequest", || ServerRequest::export_all_to(out_dir))?;
|
||||
export_ts_with_context("server responses", || export_server_responses(out_dir))?;
|
||||
export_ts_with_context("ServerNotification", || {
|
||||
ServerNotification::export_all_to(out_dir)
|
||||
})?;
|
||||
ServerRequest::export_all_to(out_dir)?;
|
||||
export_server_responses(out_dir)?;
|
||||
ServerNotification::export_all_to(out_dir)?;
|
||||
|
||||
generate_index_ts(out_dir)?;
|
||||
|
||||
|
||||
@@ -127,7 +127,7 @@ client_request_definitions! {
|
||||
#[ts(rename = "account/read")]
|
||||
GetAccount {
|
||||
params: #[ts(type = "undefined")] #[serde(skip_serializing_if = "Option::is_none")] Option<()>,
|
||||
response: GetAccountResponse,
|
||||
response: Option<Account>,
|
||||
},
|
||||
|
||||
/// DEPRECATED APIs below
|
||||
@@ -534,12 +534,6 @@ pub struct GetAccountRateLimitsResponse {
|
||||
pub rate_limits: RateLimitSnapshot,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)]
|
||||
#[serde(transparent)]
|
||||
#[ts(export)]
|
||||
#[ts(type = "Account | null")]
|
||||
pub struct GetAccountResponse(#[ts(type = "Account | null")] pub Option<Account>);
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct GetAuthStatusResponse {
|
||||
|
||||
@@ -34,7 +34,6 @@ dunce = { workspace = true }
|
||||
env-flags = { workspace = true }
|
||||
eventsource-stream = { workspace = true }
|
||||
futures = { workspace = true }
|
||||
http = { workspace = true }
|
||||
indexmap = { workspace = true }
|
||||
libc = { workspace = true }
|
||||
mcp-types = { workspace = true }
|
||||
|
||||
@@ -21,7 +21,6 @@ use codex_app_server_protocol::AuthMode;
|
||||
use codex_protocol::config_types::ForcedLoginMethod;
|
||||
|
||||
use crate::config::Config;
|
||||
use crate::default_client::CodexHttpClient;
|
||||
use crate::token_data::PlanType;
|
||||
use crate::token_data::TokenData;
|
||||
use crate::token_data::parse_id_token;
|
||||
@@ -33,7 +32,7 @@ pub struct CodexAuth {
|
||||
pub(crate) api_key: Option<String>,
|
||||
pub(crate) auth_dot_json: Arc<Mutex<Option<AuthDotJson>>>,
|
||||
pub(crate) auth_file: PathBuf,
|
||||
pub(crate) client: CodexHttpClient,
|
||||
pub(crate) client: reqwest::Client,
|
||||
}
|
||||
|
||||
impl PartialEq for CodexAuth {
|
||||
@@ -44,8 +43,6 @@ impl PartialEq for CodexAuth {
|
||||
|
||||
impl CodexAuth {
|
||||
pub async fn refresh_token(&self) -> Result<String, std::io::Error> {
|
||||
tracing::info!("Refreshing token");
|
||||
|
||||
let token_data = self
|
||||
.get_current_token_data()
|
||||
.ok_or(std::io::Error::other("Token data is not available."))?;
|
||||
@@ -183,7 +180,7 @@ impl CodexAuth {
|
||||
}
|
||||
}
|
||||
|
||||
fn from_api_key_with_client(api_key: &str, client: CodexHttpClient) -> Self {
|
||||
fn from_api_key_with_client(api_key: &str, client: reqwest::Client) -> Self {
|
||||
Self {
|
||||
api_key: Some(api_key.to_owned()),
|
||||
mode: AuthMode::ApiKey,
|
||||
@@ -403,7 +400,7 @@ async fn update_tokens(
|
||||
|
||||
async fn try_refresh_token(
|
||||
refresh_token: String,
|
||||
client: &CodexHttpClient,
|
||||
client: &reqwest::Client,
|
||||
) -> std::io::Result<RefreshResponse> {
|
||||
let refresh_request = RefreshRequest {
|
||||
client_id: CLIENT_ID,
|
||||
@@ -919,10 +916,7 @@ impl AuthManager {
|
||||
self.reload();
|
||||
Ok(Some(token))
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::error!("Failed to refresh token: {}", e);
|
||||
Err(e)
|
||||
}
|
||||
Err(e) => Err(e),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -4,7 +4,6 @@ use crate::ModelProviderInfo;
|
||||
use crate::client_common::Prompt;
|
||||
use crate::client_common::ResponseEvent;
|
||||
use crate::client_common::ResponseStream;
|
||||
use crate::default_client::CodexHttpClient;
|
||||
use crate::error::CodexErr;
|
||||
use crate::error::ConnectionFailedError;
|
||||
use crate::error::ResponseStreamFailed;
|
||||
@@ -37,7 +36,7 @@ use tracing::trace;
|
||||
pub(crate) async fn stream_chat_completions(
|
||||
prompt: &Prompt,
|
||||
model_family: &ModelFamily,
|
||||
client: &CodexHttpClient,
|
||||
client: &reqwest::Client,
|
||||
provider: &ModelProviderInfo,
|
||||
otel_event_manager: &OtelEventManager,
|
||||
) -> Result<ResponseStream> {
|
||||
|
||||
@@ -39,7 +39,6 @@ use crate::client_common::ResponsesApiRequest;
|
||||
use crate::client_common::create_reasoning_param_for_request;
|
||||
use crate::client_common::create_text_param_for_request;
|
||||
use crate::config::Config;
|
||||
use crate::default_client::CodexHttpClient;
|
||||
use crate::default_client::create_client;
|
||||
use crate::error::CodexErr;
|
||||
use crate::error::ConnectionFailedError;
|
||||
@@ -82,7 +81,7 @@ pub struct ModelClient {
|
||||
config: Arc<Config>,
|
||||
auth_manager: Option<Arc<AuthManager>>,
|
||||
otel_event_manager: OtelEventManager,
|
||||
client: CodexHttpClient,
|
||||
client: reqwest::Client,
|
||||
provider: ModelProviderInfo,
|
||||
conversation_id: ConversationId,
|
||||
effort: Option<ReasoningEffortConfig>,
|
||||
@@ -301,7 +300,6 @@ impl ModelClient {
|
||||
"POST to {}: {:?}",
|
||||
self.provider.get_full_url(&auth),
|
||||
serde_json::to_string(payload_json)
|
||||
.unwrap_or("<unable to serialize payload>".to_string())
|
||||
);
|
||||
|
||||
let mut req_builder = self
|
||||
@@ -337,6 +335,13 @@ impl ModelClient {
|
||||
.headers()
|
||||
.get("cf-ray")
|
||||
.map(|v| v.to_str().unwrap_or_default().to_string());
|
||||
|
||||
debug!(
|
||||
"Response status: {}, cf-ray: {:?}, version: {:?}",
|
||||
resp.status(),
|
||||
request_id,
|
||||
resp.version()
|
||||
);
|
||||
}
|
||||
|
||||
match res {
|
||||
|
||||
@@ -8,7 +8,6 @@ use crate::AuthManager;
|
||||
use crate::client_common::REVIEW_PROMPT;
|
||||
use crate::function_tool::FunctionCallError;
|
||||
use crate::mcp::auth::McpAuthStatusEntry;
|
||||
use crate::mcp_connection_manager::DEFAULT_STARTUP_TIMEOUT;
|
||||
use crate::parse_command::parse_command;
|
||||
use crate::parse_turn_item;
|
||||
use crate::response_processing::process_items;
|
||||
@@ -59,7 +58,8 @@ use crate::client_common::ResponseEvent;
|
||||
use crate::config::Config;
|
||||
use crate::config_types::McpServerTransportConfig;
|
||||
use crate::config_types::ShellEnvironmentPolicy;
|
||||
use crate::context_manager::ContextManager;
|
||||
use crate::conversation_history::ConversationHistory;
|
||||
use crate::conversation_history::prefetch_tokenizer_in_background;
|
||||
use crate::environment_context::EnvironmentContext;
|
||||
use crate::error::CodexErr;
|
||||
use crate::error::Result as CodexResult;
|
||||
@@ -160,6 +160,8 @@ impl Codex {
|
||||
conversation_history: InitialHistory,
|
||||
session_source: SessionSource,
|
||||
) -> CodexResult<CodexSpawnOk> {
|
||||
// Start loading the tokenizer in the background so we don't block later.
|
||||
prefetch_tokenizer_in_background();
|
||||
let (tx_sub, rx_sub) = async_channel::bounded(SUBMISSION_CHANNEL_CAPACITY);
|
||||
let (tx_event, rx_event) = async_channel::unbounded();
|
||||
|
||||
@@ -569,7 +571,9 @@ impl Session {
|
||||
// Dispatch the SessionConfiguredEvent first and then report any errors.
|
||||
// If resuming, include converted initial messages in the payload so UIs can render them immediately.
|
||||
let initial_messages = initial_history.get_event_msgs();
|
||||
sess.record_initial_history(initial_history).await;
|
||||
sess.record_initial_history(initial_history)
|
||||
.await
|
||||
.map_err(anyhow::Error::new)?;
|
||||
|
||||
let events = std::iter::once(Event {
|
||||
id: INITIAL_SUBMIT_ID.to_owned(),
|
||||
@@ -602,13 +606,16 @@ impl Session {
|
||||
format!("auto-compact-{id}")
|
||||
}
|
||||
|
||||
async fn record_initial_history(&self, conversation_history: InitialHistory) {
|
||||
async fn record_initial_history(
|
||||
&self,
|
||||
conversation_history: InitialHistory,
|
||||
) -> CodexResult<()> {
|
||||
let turn_context = self.new_turn(SessionSettingsUpdate::default()).await;
|
||||
match conversation_history {
|
||||
InitialHistory::New => {
|
||||
// Build and record initial items (user instructions + environment context)
|
||||
let items = self.build_initial_context(&turn_context);
|
||||
self.record_conversation_items(&items).await;
|
||||
self.record_conversation_items(&items).await?;
|
||||
}
|
||||
InitialHistory::Resumed(_) | InitialHistory::Forked(_) => {
|
||||
let rollout_items = conversation_history.get_rollout_items();
|
||||
@@ -616,9 +623,9 @@ impl Session {
|
||||
|
||||
// Always add response items to conversation history
|
||||
let reconstructed_history =
|
||||
self.reconstruct_history_from_rollout(&turn_context, &rollout_items);
|
||||
self.reconstruct_history_from_rollout(&turn_context, &rollout_items)?;
|
||||
if !reconstructed_history.is_empty() {
|
||||
self.record_into_history(&reconstructed_history).await;
|
||||
self.record_into_history(&reconstructed_history).await?;
|
||||
}
|
||||
|
||||
// If persisting, persist all rollout items as-is (recorder filters)
|
||||
@@ -627,6 +634,7 @@ impl Session {
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) async fn update_settings(&self, updates: SessionSettingsUpdate) {
|
||||
@@ -857,21 +865,25 @@ impl Session {
|
||||
|
||||
/// Records input items: always append to conversation history and
|
||||
/// persist these response items to rollout.
|
||||
pub(crate) async fn record_conversation_items(&self, items: &[ResponseItem]) {
|
||||
self.record_into_history(items).await;
|
||||
pub(crate) async fn record_conversation_items(
|
||||
&self,
|
||||
items: &[ResponseItem],
|
||||
) -> CodexResult<()> {
|
||||
self.record_into_history(items).await?;
|
||||
self.persist_rollout_response_items(items).await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn reconstruct_history_from_rollout(
|
||||
&self,
|
||||
turn_context: &TurnContext,
|
||||
rollout_items: &[RolloutItem],
|
||||
) -> Vec<ResponseItem> {
|
||||
let mut history = ContextManager::new();
|
||||
) -> CodexResult<Vec<ResponseItem>> {
|
||||
let mut history = ConversationHistory::new();
|
||||
for item in rollout_items {
|
||||
match item {
|
||||
RolloutItem::ResponseItem(response_item) => {
|
||||
history.record_items(std::iter::once(response_item));
|
||||
history.record_items(std::iter::once(response_item))?;
|
||||
}
|
||||
RolloutItem::Compacted(compacted) => {
|
||||
let snapshot = history.get_history();
|
||||
@@ -886,13 +898,14 @@ impl Session {
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
history.get_history()
|
||||
Ok(history.get_history())
|
||||
}
|
||||
|
||||
/// Append ResponseItems to the in-memory conversation history only.
|
||||
async fn record_into_history(&self, items: &[ResponseItem]) {
|
||||
async fn record_into_history(&self, items: &[ResponseItem]) -> CodexResult<()> {
|
||||
let mut state = self.state.lock().await;
|
||||
state.record_items(items.iter());
|
||||
state.record_items(items.iter())?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn replace_history(&self, items: Vec<ResponseItem>) {
|
||||
@@ -941,7 +954,7 @@ impl Session {
|
||||
state.history_snapshot()
|
||||
}
|
||||
|
||||
pub(crate) async fn clone_history(&self) -> ContextManager {
|
||||
pub(crate) async fn clone_history(&self) -> ConversationHistory {
|
||||
let state = self.state.lock().await;
|
||||
state.clone_history()
|
||||
}
|
||||
@@ -1001,11 +1014,11 @@ impl Session {
|
||||
&self,
|
||||
turn_context: &TurnContext,
|
||||
response_input: &ResponseInputItem,
|
||||
) {
|
||||
) -> CodexResult<()> {
|
||||
let response_item: ResponseItem = response_input.clone().into();
|
||||
// Add to conversation history and persist response item to rollout
|
||||
self.record_conversation_items(std::slice::from_ref(&response_item))
|
||||
.await;
|
||||
.await?;
|
||||
|
||||
// Derive user message events and persist only UserMessage to rollout
|
||||
let turn_item = parse_turn_item(&response_item);
|
||||
@@ -1014,6 +1027,7 @@ impl Session {
|
||||
self.emit_turn_item_started_completed(turn_context, item, false)
|
||||
.await;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Helper that emits a BackgroundEvent with the given message. This keeps
|
||||
@@ -1194,9 +1208,17 @@ async fn submission_loop(sess: Arc<Session>, config: Arc<Config>, rx_sub: Receiv
|
||||
if let Err(items) = sess.inject_input(items).await {
|
||||
if let Some(env_item) = sess
|
||||
.build_environment_update_item(previous_context.as_ref(), ¤t_context)
|
||||
&& let Err(err) = sess
|
||||
.record_conversation_items(std::slice::from_ref(&env_item))
|
||||
.await
|
||||
{
|
||||
sess.record_conversation_items(std::slice::from_ref(&env_item))
|
||||
.await;
|
||||
sess.send_event(
|
||||
current_context.as_ref(),
|
||||
EventMsg::Error(ErrorEvent {
|
||||
message: err.to_string(),
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
sess.spawn_task(Arc::clone(¤t_context), items, RegularTask)
|
||||
@@ -1509,9 +1531,9 @@ pub(crate) async fn run_task(
|
||||
input: Vec<UserInput>,
|
||||
task_kind: TaskKind,
|
||||
cancellation_token: CancellationToken,
|
||||
) -> Option<String> {
|
||||
) -> CodexResult<Option<String>> {
|
||||
if input.is_empty() {
|
||||
return None;
|
||||
return Ok(None);
|
||||
}
|
||||
let event = EventMsg::TaskStarted(TaskStartedEvent {
|
||||
model_context_window: turn_context.client.get_model_context_window(),
|
||||
@@ -1524,15 +1546,15 @@ pub(crate) async fn run_task(
|
||||
// For normal turns, continue recording to the session history as before.
|
||||
let is_review_mode = turn_context.is_review_mode;
|
||||
|
||||
let mut review_thread_history: ContextManager = ContextManager::new();
|
||||
let mut review_thread_history: ConversationHistory = ConversationHistory::new();
|
||||
if is_review_mode {
|
||||
// Seed review threads with environment context so the model knows the working directory.
|
||||
review_thread_history
|
||||
.record_items(sess.build_initial_context(turn_context.as_ref()).iter());
|
||||
review_thread_history.record_items(std::iter::once(&initial_input_for_turn.into()));
|
||||
.record_items(sess.build_initial_context(turn_context.as_ref()).iter())?;
|
||||
review_thread_history.record_items(std::iter::once(&initial_input_for_turn.into()))?;
|
||||
} else {
|
||||
sess.record_input_and_rollout_usermsg(turn_context.as_ref(), &initial_input_for_turn)
|
||||
.await;
|
||||
.await?;
|
||||
}
|
||||
|
||||
let mut last_agent_message: Option<String> = None;
|
||||
@@ -1564,11 +1586,11 @@ pub(crate) async fn run_task(
|
||||
// represents an append-only log without duplicates.
|
||||
let turn_input: Vec<ResponseItem> = if is_review_mode {
|
||||
if !pending_input.is_empty() {
|
||||
review_thread_history.record_items(&pending_input);
|
||||
review_thread_history.record_items(&pending_input)?;
|
||||
}
|
||||
review_thread_history.get_history()
|
||||
} else {
|
||||
sess.record_conversation_items(&pending_input).await;
|
||||
sess.record_conversation_items(&pending_input).await?;
|
||||
sess.history_snapshot().await
|
||||
};
|
||||
|
||||
@@ -1616,7 +1638,7 @@ pub(crate) async fn run_task(
|
||||
&mut review_thread_history,
|
||||
&sess,
|
||||
)
|
||||
.await;
|
||||
.await?;
|
||||
|
||||
if token_limit_reached {
|
||||
if auto_compact_recently_attempted {
|
||||
@@ -1633,7 +1655,8 @@ pub(crate) async fn run_task(
|
||||
break;
|
||||
}
|
||||
auto_compact_recently_attempted = true;
|
||||
compact::run_inline_auto_compact_task(sess.clone(), turn_context.clone()).await;
|
||||
compact::run_inline_auto_compact_task(sess.clone(), turn_context.clone())
|
||||
.await?;
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -1658,13 +1681,13 @@ pub(crate) async fn run_task(
|
||||
Err(CodexErr::TurnAborted {
|
||||
dangling_artifacts: processed_items,
|
||||
}) => {
|
||||
let _ = process_items(
|
||||
process_items(
|
||||
processed_items,
|
||||
is_review_mode,
|
||||
&mut review_thread_history,
|
||||
&sess,
|
||||
)
|
||||
.await;
|
||||
.await?;
|
||||
// Aborted turn is reported via a different event.
|
||||
break;
|
||||
}
|
||||
@@ -1693,10 +1716,10 @@ pub(crate) async fn run_task(
|
||||
Arc::clone(&turn_context),
|
||||
last_agent_message.as_deref().map(parse_review_output_event),
|
||||
)
|
||||
.await;
|
||||
.await?;
|
||||
}
|
||||
|
||||
last_agent_message
|
||||
Ok(last_agent_message)
|
||||
}
|
||||
|
||||
/// Parse the review output; when not valid JSON, build a structured
|
||||
@@ -2135,7 +2158,7 @@ pub(crate) async fn exit_review_mode(
|
||||
session: Arc<Session>,
|
||||
turn_context: Arc<TurnContext>,
|
||||
review_output: Option<ReviewOutputEvent>,
|
||||
) {
|
||||
) -> CodexResult<()> {
|
||||
let event = EventMsg::ExitedReviewMode(ExitedReviewModeEvent {
|
||||
review_output: review_output.clone(),
|
||||
});
|
||||
@@ -2178,7 +2201,8 @@ pub(crate) async fn exit_review_mode(
|
||||
role: "user".to_string(),
|
||||
content: vec![ContentItem::InputText { text: user_message }],
|
||||
}])
|
||||
.await;
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn mcp_init_error_display(
|
||||
@@ -2200,24 +2224,12 @@ fn mcp_init_error_display(
|
||||
// That means that the user has to specify a personal access token either via bearer_token_env_var or http_headers.
|
||||
// https://github.com/github/github-mcp-server/issues/921#issuecomment-3221026448
|
||||
format!(
|
||||
"GitHub MCP does not support OAuth. Log in by adding a personal access token (https://github.com/settings/personal-access-tokens) to your environment and config.toml:\n[mcp_servers.{server_name}]\nbearer_token_env_var = CODEX_GITHUB_PERSONAL_ACCESS_TOKEN"
|
||||
"GitHub MCP does not support OAuth. Log in by adding `bearer_token_env_var = CODEX_GITHUB_PAT` in the `mcp_servers.{server_name}` section of your config.toml"
|
||||
)
|
||||
} else if is_mcp_client_auth_required_error(err) {
|
||||
format!(
|
||||
"The {server_name} MCP server is not logged in. Run `codex mcp login {server_name}`."
|
||||
)
|
||||
} else if is_mcp_client_startup_timeout_error(err) {
|
||||
let startup_timeout_secs = match entry {
|
||||
Some(entry) => match entry.config.startup_timeout_sec {
|
||||
Some(timeout) => timeout,
|
||||
None => DEFAULT_STARTUP_TIMEOUT,
|
||||
},
|
||||
None => DEFAULT_STARTUP_TIMEOUT,
|
||||
}
|
||||
.as_secs();
|
||||
format!(
|
||||
"MCP client for `{server_name}` timed out after {startup_timeout_secs} seconds. Add or adjust `startup_timeout_sec` in your config.toml:\n[mcp_servers.{server_name}]\nstartup_timeout_sec = XX"
|
||||
)
|
||||
} else {
|
||||
format!("MCP client for `{server_name}` failed to start: {err:#}")
|
||||
}
|
||||
@@ -2228,12 +2240,6 @@ fn is_mcp_client_auth_required_error(error: &anyhow::Error) -> bool {
|
||||
error.to_string().contains("Auth required")
|
||||
}
|
||||
|
||||
fn is_mcp_client_startup_timeout_error(error: &anyhow::Error) -> bool {
|
||||
let error_message = error.to_string();
|
||||
error_message.contains("request timed out")
|
||||
|| error_message.contains("timed out handshaking with MCP server")
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) use tests::make_session_and_context;
|
||||
|
||||
@@ -2244,6 +2250,7 @@ mod tests {
|
||||
use crate::config::ConfigToml;
|
||||
use crate::config_types::McpServerConfig;
|
||||
use crate::config_types::McpServerTransportConfig;
|
||||
use crate::error::Result as CodexResult;
|
||||
use crate::exec::ExecToolCallOutput;
|
||||
use crate::mcp::auth::McpAuthStatusEntry;
|
||||
use crate::tools::format_exec_output_str;
|
||||
@@ -2284,9 +2291,12 @@ mod tests {
|
||||
#[test]
|
||||
fn reconstruct_history_matches_live_compactions() {
|
||||
let (session, turn_context) = make_session_and_context();
|
||||
let (rollout_items, expected) = sample_rollout(&session, &turn_context);
|
||||
let (rollout_items, expected) =
|
||||
sample_rollout(&session, &turn_context).expect("sample rollout");
|
||||
|
||||
let reconstructed = session.reconstruct_history_from_rollout(&turn_context, &rollout_items);
|
||||
let reconstructed = session
|
||||
.reconstruct_history_from_rollout(&turn_context, &rollout_items)
|
||||
.expect("reconstruct history");
|
||||
|
||||
assert_eq!(expected, reconstructed);
|
||||
}
|
||||
@@ -2294,15 +2304,19 @@ mod tests {
|
||||
#[test]
|
||||
fn record_initial_history_reconstructs_resumed_transcript() {
|
||||
let (session, turn_context) = make_session_and_context();
|
||||
let (rollout_items, expected) = sample_rollout(&session, &turn_context);
|
||||
let (rollout_items, expected) =
|
||||
sample_rollout(&session, &turn_context).expect("sample rollout");
|
||||
|
||||
tokio_test::block_on(session.record_initial_history(InitialHistory::Resumed(
|
||||
ResumedHistory {
|
||||
conversation_id: ConversationId::default(),
|
||||
history: rollout_items,
|
||||
rollout_path: PathBuf::from("/tmp/resume.jsonl"),
|
||||
},
|
||||
)));
|
||||
tokio_test::block_on(async {
|
||||
session
|
||||
.record_initial_history(InitialHistory::Resumed(ResumedHistory {
|
||||
conversation_id: ConversationId::default(),
|
||||
history: rollout_items,
|
||||
rollout_path: PathBuf::from("/tmp/resume.jsonl"),
|
||||
}))
|
||||
.await
|
||||
.expect("record resumed history");
|
||||
});
|
||||
|
||||
let actual = tokio_test::block_on(async { session.state.lock().await.history_snapshot() });
|
||||
assert_eq!(expected, actual);
|
||||
@@ -2311,9 +2325,15 @@ mod tests {
|
||||
#[test]
|
||||
fn record_initial_history_reconstructs_forked_transcript() {
|
||||
let (session, turn_context) = make_session_and_context();
|
||||
let (rollout_items, expected) = sample_rollout(&session, &turn_context);
|
||||
let (rollout_items, expected) =
|
||||
sample_rollout(&session, &turn_context).expect("sample rollout");
|
||||
|
||||
tokio_test::block_on(session.record_initial_history(InitialHistory::Forked(rollout_items)));
|
||||
tokio_test::block_on(async {
|
||||
session
|
||||
.record_initial_history(InitialHistory::Forked(rollout_items))
|
||||
.await
|
||||
.expect("record forked history");
|
||||
});
|
||||
|
||||
let actual = tokio_test::block_on(async { session.state.lock().await.history_snapshot() });
|
||||
assert_eq!(expected, actual);
|
||||
@@ -2673,10 +2693,10 @@ mod tests {
|
||||
_ctx: Arc<TurnContext>,
|
||||
_input: Vec<UserInput>,
|
||||
cancellation_token: CancellationToken,
|
||||
) -> Option<String> {
|
||||
) -> CodexResult<Option<String>> {
|
||||
if self.listen_to_cancellation_token {
|
||||
cancellation_token.cancelled().await;
|
||||
return None;
|
||||
return Ok(None);
|
||||
}
|
||||
loop {
|
||||
sleep(Duration::from_secs(60)).await;
|
||||
@@ -2685,7 +2705,7 @@ mod tests {
|
||||
|
||||
async fn abort(&self, session: Arc<SessionTaskContext>, ctx: Arc<TurnContext>) {
|
||||
if let TaskKind::Review = self.kind {
|
||||
exit_review_mode(session.clone_session(), ctx, None).await;
|
||||
let _ = exit_review_mode(session.clone_session(), ctx, None).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2841,15 +2861,15 @@ mod tests {
|
||||
fn sample_rollout(
|
||||
session: &Session,
|
||||
turn_context: &TurnContext,
|
||||
) -> (Vec<RolloutItem>, Vec<ResponseItem>) {
|
||||
) -> CodexResult<(Vec<RolloutItem>, Vec<ResponseItem>)> {
|
||||
let mut rollout_items = Vec::new();
|
||||
let mut live_history = ContextManager::new();
|
||||
let mut live_history = ConversationHistory::new();
|
||||
|
||||
let initial_context = session.build_initial_context(turn_context);
|
||||
for item in &initial_context {
|
||||
rollout_items.push(RolloutItem::ResponseItem(item.clone()));
|
||||
}
|
||||
live_history.record_items(initial_context.iter());
|
||||
live_history.record_items(initial_context.iter())?;
|
||||
|
||||
let user1 = ResponseItem::Message {
|
||||
id: None,
|
||||
@@ -2858,7 +2878,7 @@ mod tests {
|
||||
text: "first user".to_string(),
|
||||
}],
|
||||
};
|
||||
live_history.record_items(std::iter::once(&user1));
|
||||
live_history.record_items(std::iter::once(&user1))?;
|
||||
rollout_items.push(RolloutItem::ResponseItem(user1.clone()));
|
||||
|
||||
let assistant1 = ResponseItem::Message {
|
||||
@@ -2868,7 +2888,7 @@ mod tests {
|
||||
text: "assistant reply one".to_string(),
|
||||
}],
|
||||
};
|
||||
live_history.record_items(std::iter::once(&assistant1));
|
||||
live_history.record_items(std::iter::once(&assistant1))?;
|
||||
rollout_items.push(RolloutItem::ResponseItem(assistant1.clone()));
|
||||
|
||||
let summary1 = "summary one";
|
||||
@@ -2891,7 +2911,7 @@ mod tests {
|
||||
text: "second user".to_string(),
|
||||
}],
|
||||
};
|
||||
live_history.record_items(std::iter::once(&user2));
|
||||
live_history.record_items(std::iter::once(&user2))?;
|
||||
rollout_items.push(RolloutItem::ResponseItem(user2.clone()));
|
||||
|
||||
let assistant2 = ResponseItem::Message {
|
||||
@@ -2901,7 +2921,7 @@ mod tests {
|
||||
text: "assistant reply two".to_string(),
|
||||
}],
|
||||
};
|
||||
live_history.record_items(std::iter::once(&assistant2));
|
||||
live_history.record_items(std::iter::once(&assistant2))?;
|
||||
rollout_items.push(RolloutItem::ResponseItem(assistant2.clone()));
|
||||
|
||||
let summary2 = "summary two";
|
||||
@@ -2924,7 +2944,7 @@ mod tests {
|
||||
text: "third user".to_string(),
|
||||
}],
|
||||
};
|
||||
live_history.record_items(std::iter::once(&user3));
|
||||
live_history.record_items(std::iter::once(&user3))?;
|
||||
rollout_items.push(RolloutItem::ResponseItem(user3.clone()));
|
||||
|
||||
let assistant3 = ResponseItem::Message {
|
||||
@@ -2934,10 +2954,10 @@ mod tests {
|
||||
text: "assistant reply three".to_string(),
|
||||
}],
|
||||
};
|
||||
live_history.record_items(std::iter::once(&assistant3));
|
||||
live_history.record_items(std::iter::once(&assistant3))?;
|
||||
rollout_items.push(RolloutItem::ResponseItem(assistant3.clone()));
|
||||
|
||||
(rollout_items, live_history.get_history())
|
||||
Ok((rollout_items, live_history.get_history()))
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -3091,7 +3111,7 @@ mod tests {
|
||||
let display = mcp_init_error_display(server_name, Some(&entry), &err);
|
||||
|
||||
let expected = format!(
|
||||
"GitHub MCP does not support OAuth. Log in by adding a personal access token (https://github.com/settings/personal-access-tokens) to your environment and config.toml:\n[mcp_servers.{server_name}]\nbearer_token_env_var = CODEX_GITHUB_PERSONAL_ACCESS_TOKEN"
|
||||
"GitHub MCP does not support OAuth. Log in by adding `bearer_token_env_var = CODEX_GITHUB_PAT` in the `mcp_servers.{server_name}` section of your config.toml"
|
||||
);
|
||||
|
||||
assert_eq!(expected, display);
|
||||
@@ -3138,17 +3158,4 @@ mod tests {
|
||||
|
||||
assert_eq!(expected, display);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mcp_init_error_display_includes_startup_timeout_hint() {
|
||||
let server_name = "slow";
|
||||
let err = anyhow::anyhow!("request timed out");
|
||||
|
||||
let display = mcp_init_error_display(server_name, None, &err);
|
||||
|
||||
assert_eq!(
|
||||
"MCP client for `slow` timed out after 10 seconds. Add or adjust `startup_timeout_sec` in your config.toml:\n[mcp_servers.slow]\nstartup_timeout_sec = XX",
|
||||
display
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -39,35 +39,35 @@ struct HistoryBridgeTemplate<'a> {
|
||||
pub(crate) async fn run_inline_auto_compact_task(
|
||||
sess: Arc<Session>,
|
||||
turn_context: Arc<TurnContext>,
|
||||
) {
|
||||
) -> CodexResult<()> {
|
||||
let input = vec![UserInput::Text {
|
||||
text: SUMMARIZATION_PROMPT.to_string(),
|
||||
}];
|
||||
run_compact_task_inner(sess, turn_context, input).await;
|
||||
run_compact_task_inner(sess, turn_context, input).await
|
||||
}
|
||||
|
||||
pub(crate) async fn run_compact_task(
|
||||
sess: Arc<Session>,
|
||||
turn_context: Arc<TurnContext>,
|
||||
input: Vec<UserInput>,
|
||||
) -> Option<String> {
|
||||
) -> CodexResult<Option<String>> {
|
||||
let start_event = EventMsg::TaskStarted(TaskStartedEvent {
|
||||
model_context_window: turn_context.client.get_model_context_window(),
|
||||
});
|
||||
sess.send_event(&turn_context, start_event).await;
|
||||
run_compact_task_inner(sess.clone(), turn_context, input).await;
|
||||
None
|
||||
run_compact_task_inner(sess.clone(), turn_context, input).await?;
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
async fn run_compact_task_inner(
|
||||
sess: Arc<Session>,
|
||||
turn_context: Arc<TurnContext>,
|
||||
input: Vec<UserInput>,
|
||||
) {
|
||||
) -> CodexResult<()> {
|
||||
let initial_input_for_turn: ResponseInputItem = ResponseInputItem::from(input);
|
||||
|
||||
let mut history = sess.clone_history().await;
|
||||
history.record_items(&[initial_input_for_turn.into()]);
|
||||
history.record_items(&[initial_input_for_turn.into()])?;
|
||||
|
||||
let mut truncated_count = 0usize;
|
||||
|
||||
@@ -106,7 +106,7 @@ async fn run_compact_task_inner(
|
||||
break;
|
||||
}
|
||||
Err(CodexErr::Interrupted) => {
|
||||
return;
|
||||
return Ok(());
|
||||
}
|
||||
Err(e @ CodexErr::ContextWindowExceeded) => {
|
||||
if turn_input.len() > 1 {
|
||||
@@ -124,7 +124,7 @@ async fn run_compact_task_inner(
|
||||
message: e.to_string(),
|
||||
});
|
||||
sess.send_event(&turn_context, event).await;
|
||||
return;
|
||||
return Ok(());
|
||||
}
|
||||
Err(e) => {
|
||||
if retries < max_retries {
|
||||
@@ -142,7 +142,7 @@ async fn run_compact_task_inner(
|
||||
message: e.to_string(),
|
||||
});
|
||||
sess.send_event(&turn_context, event).await;
|
||||
return;
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -164,6 +164,7 @@ async fn run_compact_task_inner(
|
||||
message: "Compact task completed".to_string(),
|
||||
});
|
||||
sess.send_event(&turn_context, event).await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn content_items_to_text(content: &[ContentItem]) -> Option<String> {
|
||||
@@ -252,7 +253,8 @@ async fn drain_to_completed(
|
||||
};
|
||||
match event {
|
||||
Ok(ResponseEvent::OutputItemDone(item)) => {
|
||||
sess.record_into_history(std::slice::from_ref(&item)).await;
|
||||
sess.record_into_history(std::slice::from_ref(&item))
|
||||
.await?;
|
||||
}
|
||||
Ok(ResponseEvent::RateLimits(snapshot)) => {
|
||||
sess.update_rate_limits(turn_context, snapshot).await;
|
||||
|
||||
@@ -1,3 +0,0 @@
|
||||
mod manager;
|
||||
pub(crate) use manager::ContextManager;
|
||||
pub mod truncation;
|
||||
@@ -1,159 +0,0 @@
|
||||
use codex_utils_string::take_bytes_at_char_boundary;
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
pub(crate) struct TruncationConfig {
|
||||
pub max_bytes: usize,
|
||||
pub max_lines: usize,
|
||||
pub truncation_notice: &'static str,
|
||||
}
|
||||
|
||||
// Telemetry preview limits: keep log events smaller than model budgets.
|
||||
pub(crate) const TELEMETRY_PREVIEW_MAX_BYTES: usize = 2 * 1024; // 2 KiB
|
||||
pub(crate) const TELEMETRY_PREVIEW_MAX_LINES: usize = 64; // lines
|
||||
pub(crate) const TELEMETRY_PREVIEW_TRUNCATION_NOTICE: &str =
|
||||
"[... telemetry preview truncated ...]";
|
||||
|
||||
pub(crate) const CONTEXT_OUTPUT_TRUNCATION: TruncationConfig = TruncationConfig {
|
||||
max_bytes: TELEMETRY_PREVIEW_MAX_BYTES,
|
||||
max_lines: TELEMETRY_PREVIEW_MAX_LINES,
|
||||
truncation_notice: TELEMETRY_PREVIEW_TRUNCATION_NOTICE,
|
||||
};
|
||||
|
||||
pub(crate) fn truncate_with_config(content: &str, config: TruncationConfig) -> String {
|
||||
let TruncationConfig {
|
||||
max_bytes,
|
||||
max_lines,
|
||||
truncation_notice,
|
||||
} = config;
|
||||
|
||||
let truncated_slice = take_bytes_at_char_boundary(content, max_bytes);
|
||||
let truncated_by_bytes = truncated_slice.len() < content.len();
|
||||
|
||||
let mut preview = String::new();
|
||||
let mut lines_iter = truncated_slice.lines();
|
||||
for idx in 0..max_lines {
|
||||
match lines_iter.next() {
|
||||
Some(line) => {
|
||||
if idx > 0 {
|
||||
preview.push('\n');
|
||||
}
|
||||
preview.push_str(line);
|
||||
}
|
||||
None => break,
|
||||
}
|
||||
}
|
||||
let truncated_by_lines = lines_iter.next().is_some();
|
||||
|
||||
if !truncated_by_bytes && !truncated_by_lines {
|
||||
return content.to_string();
|
||||
}
|
||||
|
||||
if preview.len() < truncated_slice.len()
|
||||
&& truncated_slice
|
||||
.as_bytes()
|
||||
.get(preview.len())
|
||||
.is_some_and(|byte| *byte == b'\n')
|
||||
{
|
||||
preview.push('\n');
|
||||
}
|
||||
|
||||
if !preview.is_empty() && !preview.ends_with('\n') {
|
||||
preview.push('\n');
|
||||
}
|
||||
|
||||
preview.push_str(truncation_notice);
|
||||
preview
|
||||
}
|
||||
|
||||
pub(crate) fn truncate_context_output(content: &str) -> String {
|
||||
truncate_with_config(content, CONTEXT_OUTPUT_TRUNCATION)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use pretty_assertions::assert_eq;
|
||||
|
||||
#[test]
|
||||
fn truncate_with_config_returns_original_within_limits() {
|
||||
let content = "short output";
|
||||
let config = TruncationConfig {
|
||||
max_bytes: 64,
|
||||
max_lines: 5,
|
||||
truncation_notice: "[notice]",
|
||||
};
|
||||
assert_eq!(truncate_with_config(content, config), content);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn truncate_with_config_truncates_by_bytes() {
|
||||
let config = TruncationConfig {
|
||||
max_bytes: 16,
|
||||
max_lines: 10,
|
||||
truncation_notice: "[notice]",
|
||||
};
|
||||
let content = "abcdefghijklmnopqrstuvwxyz";
|
||||
let truncated = truncate_with_config(content, config);
|
||||
assert!(truncated.contains("[notice]"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn truncate_with_config_truncates_by_lines() {
|
||||
let config = TruncationConfig {
|
||||
max_bytes: 1024,
|
||||
max_lines: 2,
|
||||
truncation_notice: "[notice]",
|
||||
};
|
||||
let content = "l1\nl2\nl3\nl4";
|
||||
let truncated = truncate_with_config(content, config);
|
||||
assert!(truncated.lines().count() <= 3);
|
||||
assert!(truncated.contains("[notice]"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn telemetry_preview_returns_original_within_limits() {
|
||||
let content = "short output";
|
||||
let config = TruncationConfig {
|
||||
max_bytes: TELEMETRY_PREVIEW_MAX_BYTES,
|
||||
max_lines: TELEMETRY_PREVIEW_MAX_LINES,
|
||||
truncation_notice: TELEMETRY_PREVIEW_TRUNCATION_NOTICE,
|
||||
};
|
||||
assert_eq!(truncate_with_config(content, config), content);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn telemetry_preview_truncates_by_bytes() {
|
||||
let config = TruncationConfig {
|
||||
max_bytes: TELEMETRY_PREVIEW_MAX_BYTES,
|
||||
max_lines: TELEMETRY_PREVIEW_MAX_LINES,
|
||||
truncation_notice: TELEMETRY_PREVIEW_TRUNCATION_NOTICE,
|
||||
};
|
||||
let content = "x".repeat(TELEMETRY_PREVIEW_MAX_BYTES + 8);
|
||||
let preview = truncate_with_config(&content, config);
|
||||
|
||||
assert!(preview.contains(TELEMETRY_PREVIEW_TRUNCATION_NOTICE));
|
||||
assert!(
|
||||
preview.len()
|
||||
<= TELEMETRY_PREVIEW_MAX_BYTES + TELEMETRY_PREVIEW_TRUNCATION_NOTICE.len() + 1
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn telemetry_preview_truncates_by_lines() {
|
||||
let config = TruncationConfig {
|
||||
max_bytes: TELEMETRY_PREVIEW_MAX_BYTES,
|
||||
max_lines: TELEMETRY_PREVIEW_MAX_LINES,
|
||||
truncation_notice: TELEMETRY_PREVIEW_TRUNCATION_NOTICE,
|
||||
};
|
||||
let content = (0..(TELEMETRY_PREVIEW_MAX_LINES + 5))
|
||||
.map(|idx| format!("line {idx}"))
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n");
|
||||
|
||||
let preview = truncate_with_config(&content, config);
|
||||
let lines: Vec<&str> = preview.lines().collect();
|
||||
|
||||
assert!(lines.len() <= TELEMETRY_PREVIEW_MAX_LINES + 1);
|
||||
assert_eq!(lines.last(), Some(&TELEMETRY_PREVIEW_TRUNCATION_NOTICE));
|
||||
}
|
||||
}
|
||||
@@ -1,23 +1,31 @@
|
||||
use crate::context_manager::truncation::truncate_context_output;
|
||||
use std::sync::Arc;
|
||||
use std::sync::OnceLock;
|
||||
|
||||
use codex_protocol::models::FunctionCallOutputPayload;
|
||||
use codex_protocol::models::ResponseItem;
|
||||
use codex_protocol::protocol::TokenUsage;
|
||||
use codex_protocol::protocol::TokenUsageInfo;
|
||||
use codex_utils_tokenizer::Tokenizer;
|
||||
use tokio::task;
|
||||
use tracing::error;
|
||||
|
||||
static TOKENIZER: OnceLock<Option<Arc<Tokenizer>>> = OnceLock::new();
|
||||
|
||||
use crate::error::CodexErr;
|
||||
|
||||
/// Transcript of conversation history
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub(crate) struct ContextManager {
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) struct ConversationHistory {
|
||||
/// The oldest items are at the beginning of the vector.
|
||||
items: Vec<ResponseItem>,
|
||||
token_info: Option<TokenUsageInfo>,
|
||||
}
|
||||
|
||||
impl ContextManager {
|
||||
impl ConversationHistory {
|
||||
pub(crate) fn new() -> Self {
|
||||
Self {
|
||||
items: Vec::new(),
|
||||
token_info: TokenUsageInfo::new_or_append(&None, &None, None),
|
||||
token_info: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -35,7 +43,7 @@ impl ContextManager {
|
||||
}
|
||||
|
||||
/// `items` is ordered from oldest to newest.
|
||||
pub(crate) fn record_items<I>(&mut self, items: I)
|
||||
pub(crate) fn record_items<I>(&mut self, items: I) -> Result<(), CodexErr>
|
||||
where
|
||||
I: IntoIterator,
|
||||
I::Item: std::ops::Deref<Target = ResponseItem>,
|
||||
@@ -44,10 +52,10 @@ impl ContextManager {
|
||||
if !is_api_message(&item) {
|
||||
continue;
|
||||
}
|
||||
|
||||
let processed = Self::process_item(&item);
|
||||
self.items.push(processed);
|
||||
self.validate_input(&item)?;
|
||||
self.items.push(item.clone());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn get_history(&mut self) -> Vec<ResponseItem> {
|
||||
@@ -78,34 +86,70 @@ impl ContextManager {
|
||||
self.remove_orphan_outputs();
|
||||
}
|
||||
|
||||
fn process_item(item: &ResponseItem) -> ResponseItem {
|
||||
match item {
|
||||
ResponseItem::FunctionCallOutput { call_id, output } => {
|
||||
let truncated_content = truncate_context_output(output.content.as_str());
|
||||
ResponseItem::FunctionCallOutput {
|
||||
call_id: call_id.clone(),
|
||||
output: FunctionCallOutputPayload {
|
||||
content: truncated_content,
|
||||
success: output.success,
|
||||
},
|
||||
}
|
||||
}
|
||||
ResponseItem::CustomToolCallOutput { call_id, output } => {
|
||||
let truncated = truncate_context_output(output);
|
||||
ResponseItem::CustomToolCallOutput {
|
||||
call_id: call_id.clone(),
|
||||
output: truncated,
|
||||
}
|
||||
}
|
||||
_ => item.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns a clone of the contents in the transcript.
|
||||
fn contents(&self) -> Vec<ResponseItem> {
|
||||
self.items.clone()
|
||||
}
|
||||
|
||||
fn validate_input(&self, item: &ResponseItem) -> Result<(), CodexErr> {
|
||||
match item {
|
||||
ResponseItem::Message { content, .. } => {
|
||||
self.validate_input_content_item(content)?;
|
||||
Ok(())
|
||||
}
|
||||
ResponseItem::FunctionCall { .. }
|
||||
| ResponseItem::FunctionCallOutput { .. }
|
||||
| ResponseItem::CustomToolCall { .. }
|
||||
| ResponseItem::CustomToolCallOutput { .. }
|
||||
| ResponseItem::LocalShellCall { .. }
|
||||
| ResponseItem::Reasoning { .. }
|
||||
| ResponseItem::WebSearchCall { .. } => Ok(()),
|
||||
ResponseItem::Other => Err(CodexErr::InvalidInput(format!("invalid input: {item:?}"))),
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_input_content_item(
|
||||
&self,
|
||||
content: &[codex_protocol::models::ContentItem],
|
||||
) -> Result<(), CodexErr> {
|
||||
let Some(info) = &self.token_info else {
|
||||
return Ok(());
|
||||
};
|
||||
// this will intentionally not check the context for the first turn before getting this information.
|
||||
// it's acceptable tradeoff.
|
||||
let Some(context_window) = info.model_context_window else {
|
||||
return Ok(());
|
||||
};
|
||||
let tokenizer = match shared_tokenizer() {
|
||||
Some(t) => t,
|
||||
None => return Ok(()),
|
||||
};
|
||||
|
||||
let mut input_tokens: i64 = 0;
|
||||
for item in content {
|
||||
match item {
|
||||
codex_protocol::models::ContentItem::InputText { text } => {
|
||||
input_tokens += tokenizer.count(text);
|
||||
}
|
||||
codex_protocol::models::ContentItem::InputImage { .. } => {
|
||||
// no validation currently
|
||||
}
|
||||
codex_protocol::models::ContentItem::OutputText { .. } => {
|
||||
// no validation currently
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let prior_total = info.last_token_usage.total_tokens;
|
||||
let combined_tokens = prior_total.saturating_add(input_tokens);
|
||||
let threshold = context_window * 95 / 100;
|
||||
if combined_tokens > threshold {
|
||||
return Err(CodexErr::InvalidInput("input too large".to_string()));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn ensure_call_outputs_present(&mut self) {
|
||||
// Collect synthetic outputs to insert immediately after their calls.
|
||||
// Store the insertion position (index of call) alongside the item so
|
||||
@@ -131,7 +175,7 @@ impl ContextManager {
|
||||
ResponseItem::FunctionCallOutput {
|
||||
call_id: call_id.clone(),
|
||||
output: FunctionCallOutputPayload {
|
||||
content: truncate_context_output("aborted"),
|
||||
content: "aborted".to_string(),
|
||||
success: None,
|
||||
},
|
||||
},
|
||||
@@ -154,7 +198,7 @@ impl ContextManager {
|
||||
idx,
|
||||
ResponseItem::CustomToolCallOutput {
|
||||
call_id: call_id.clone(),
|
||||
output: truncate_context_output("aborted"),
|
||||
output: "aborted".to_string(),
|
||||
},
|
||||
));
|
||||
}
|
||||
@@ -178,7 +222,7 @@ impl ContextManager {
|
||||
ResponseItem::FunctionCallOutput {
|
||||
call_id: call_id.clone(),
|
||||
output: FunctionCallOutputPayload {
|
||||
content: truncate_context_output("aborted"),
|
||||
content: "aborted".to_string(),
|
||||
success: None,
|
||||
},
|
||||
},
|
||||
@@ -274,10 +318,7 @@ impl ContextManager {
|
||||
}
|
||||
|
||||
pub(crate) fn replace(&mut self, items: Vec<ResponseItem>) {
|
||||
self.items = items
|
||||
.into_iter()
|
||||
.map(|item| Self::process_item(&item))
|
||||
.collect();
|
||||
self.items = items;
|
||||
}
|
||||
|
||||
/// Removes the corresponding paired item for the provided `item`, if any.
|
||||
@@ -371,6 +412,36 @@ fn error_or_panic(message: String) {
|
||||
}
|
||||
}
|
||||
|
||||
fn shared_tokenizer() -> Option<Arc<Tokenizer>> {
|
||||
TOKENIZER.get().and_then(|opt| opt.as_ref().map(Arc::clone))
|
||||
}
|
||||
|
||||
/// Kick off background initialization of the shared tokenizer without blocking the caller.
|
||||
pub(crate) fn prefetch_tokenizer_in_background() {
|
||||
if TOKENIZER.get().is_some() {
|
||||
return;
|
||||
}
|
||||
|
||||
// Spawn a background task to initialize the tokenizer. Use spawn_blocking in case
|
||||
// initialization performs CPU-heavy work or file I/O.
|
||||
tokio::spawn(async {
|
||||
let result = task::spawn_blocking(Tokenizer::try_default).await;
|
||||
match result {
|
||||
Ok(Ok(tokenizer)) => {
|
||||
let _ = TOKENIZER.set(Some(Arc::new(tokenizer)));
|
||||
}
|
||||
Ok(Err(error)) => {
|
||||
error!("failed to create tokenizer: {error}");
|
||||
let _ = TOKENIZER.set(None);
|
||||
}
|
||||
Err(join_error) => {
|
||||
error!("failed to join tokenizer init task: {join_error}");
|
||||
let _ = TOKENIZER.set(None);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// Anything that is not a system message or "reasoning" message is considered
|
||||
/// an API message.
|
||||
fn is_api_message(message: &ResponseItem) -> bool {
|
||||
@@ -390,8 +461,6 @@ fn is_api_message(message: &ResponseItem) -> bool {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::context_manager::truncation::TELEMETRY_PREVIEW_MAX_BYTES;
|
||||
use crate::context_manager::truncation::TELEMETRY_PREVIEW_TRUNCATION_NOTICE;
|
||||
use codex_protocol::models::ContentItem;
|
||||
use codex_protocol::models::FunctionCallOutputPayload;
|
||||
use codex_protocol::models::LocalShellAction;
|
||||
@@ -409,9 +478,9 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
fn create_history_with_items(items: Vec<ResponseItem>) -> ContextManager {
|
||||
let mut h = ContextManager::new();
|
||||
h.record_items(items.iter());
|
||||
fn create_history_with_items(items: Vec<ResponseItem>) -> ConversationHistory {
|
||||
let mut h = ConversationHistory::new();
|
||||
h.record_items(items.iter()).unwrap();
|
||||
h
|
||||
}
|
||||
|
||||
@@ -427,7 +496,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn filters_non_api_messages() {
|
||||
let mut h = ContextManager::default();
|
||||
let mut h = ConversationHistory::new();
|
||||
// System message is not an API message; Other is ignored.
|
||||
let system = ResponseItem::Message {
|
||||
id: None,
|
||||
@@ -436,12 +505,12 @@ mod tests {
|
||||
text: "ignored".to_string(),
|
||||
}],
|
||||
};
|
||||
h.record_items([&system, &ResponseItem::Other]);
|
||||
h.record_items([&system, &ResponseItem::Other]).unwrap();
|
||||
|
||||
// User and assistant should be retained.
|
||||
let u = user_msg("hi");
|
||||
let a = assistant_msg("hello");
|
||||
h.record_items([&u, &a]);
|
||||
h.record_items([&u, &a]).unwrap();
|
||||
|
||||
let items = h.contents();
|
||||
assert_eq!(
|
||||
@@ -465,61 +534,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn record_items_truncates_function_call_output() {
|
||||
let mut h = ContextManager::new();
|
||||
let long_content = "a".repeat(TELEMETRY_PREVIEW_MAX_BYTES + 32);
|
||||
let item = ResponseItem::FunctionCallOutput {
|
||||
call_id: "call-long".to_string(),
|
||||
output: FunctionCallOutputPayload {
|
||||
content: long_content.clone(),
|
||||
success: Some(true),
|
||||
},
|
||||
};
|
||||
|
||||
h.record_items([&item]);
|
||||
|
||||
let stored = h.contents();
|
||||
let ResponseItem::FunctionCallOutput { output, .. } = &stored[0] else {
|
||||
panic!("expected FunctionCallOutput variant");
|
||||
};
|
||||
assert!(
|
||||
output
|
||||
.content
|
||||
.ends_with(TELEMETRY_PREVIEW_TRUNCATION_NOTICE),
|
||||
"truncated content should end with notice"
|
||||
);
|
||||
assert!(
|
||||
output.content.len() < long_content.len(),
|
||||
"content should shrink after truncation"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn record_items_truncates_custom_tool_output() {
|
||||
let mut h = ContextManager::new();
|
||||
let long_content = "b".repeat(TELEMETRY_PREVIEW_MAX_BYTES + 64);
|
||||
let item = ResponseItem::CustomToolCallOutput {
|
||||
call_id: "custom-long".to_string(),
|
||||
output: long_content.clone(),
|
||||
};
|
||||
|
||||
h.record_items([&item]);
|
||||
|
||||
let stored = h.contents();
|
||||
let ResponseItem::CustomToolCallOutput { output, .. } = &stored[0] else {
|
||||
panic!("expected CustomToolCallOutput variant");
|
||||
};
|
||||
assert!(
|
||||
output.ends_with(TELEMETRY_PREVIEW_TRUNCATION_NOTICE),
|
||||
"truncated output should end with notice"
|
||||
);
|
||||
assert!(
|
||||
output.len() < long_content.len(),
|
||||
"output should shrink after truncation"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remove_first_item_removes_matching_output_for_function_call() {
|
||||
let items = vec![
|
||||
@@ -1,13 +1,5 @@
|
||||
use crate::spawn::CODEX_SANDBOX_ENV_VAR;
|
||||
use http::Error as HttpError;
|
||||
use reqwest::IntoUrl;
|
||||
use reqwest::Method;
|
||||
use reqwest::Response;
|
||||
use reqwest::header::HeaderName;
|
||||
use reqwest::header::HeaderValue;
|
||||
use serde::Serialize;
|
||||
use std::collections::HashMap;
|
||||
use std::fmt::Display;
|
||||
use std::sync::LazyLock;
|
||||
use std::sync::Mutex;
|
||||
use std::sync::OnceLock;
|
||||
@@ -30,130 +22,6 @@ use std::sync::OnceLock;
|
||||
pub static USER_AGENT_SUFFIX: LazyLock<Mutex<Option<String>>> = LazyLock::new(|| Mutex::new(None));
|
||||
pub const DEFAULT_ORIGINATOR: &str = "codex_cli_rs";
|
||||
pub const CODEX_INTERNAL_ORIGINATOR_OVERRIDE_ENV_VAR: &str = "CODEX_INTERNAL_ORIGINATOR_OVERRIDE";
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct CodexHttpClient {
|
||||
inner: reqwest::Client,
|
||||
}
|
||||
|
||||
impl CodexHttpClient {
|
||||
fn new(inner: reqwest::Client) -> Self {
|
||||
Self { inner }
|
||||
}
|
||||
|
||||
pub fn get<U>(&self, url: U) -> CodexRequestBuilder
|
||||
where
|
||||
U: IntoUrl,
|
||||
{
|
||||
self.request(Method::GET, url)
|
||||
}
|
||||
|
||||
pub fn post<U>(&self, url: U) -> CodexRequestBuilder
|
||||
where
|
||||
U: IntoUrl,
|
||||
{
|
||||
self.request(Method::POST, url)
|
||||
}
|
||||
|
||||
pub fn request<U>(&self, method: Method, url: U) -> CodexRequestBuilder
|
||||
where
|
||||
U: IntoUrl,
|
||||
{
|
||||
let url_str = url.as_str().to_string();
|
||||
CodexRequestBuilder::new(self.inner.request(method.clone(), url), method, url_str)
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use = "requests are not sent unless `send` is awaited"]
|
||||
#[derive(Debug)]
|
||||
pub struct CodexRequestBuilder {
|
||||
builder: reqwest::RequestBuilder,
|
||||
method: Method,
|
||||
url: String,
|
||||
}
|
||||
|
||||
impl CodexRequestBuilder {
|
||||
fn new(builder: reqwest::RequestBuilder, method: Method, url: String) -> Self {
|
||||
Self {
|
||||
builder,
|
||||
method,
|
||||
url,
|
||||
}
|
||||
}
|
||||
|
||||
fn map(self, f: impl FnOnce(reqwest::RequestBuilder) -> reqwest::RequestBuilder) -> Self {
|
||||
Self {
|
||||
builder: f(self.builder),
|
||||
method: self.method,
|
||||
url: self.url,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn header<K, V>(self, key: K, value: V) -> Self
|
||||
where
|
||||
HeaderName: TryFrom<K>,
|
||||
<HeaderName as TryFrom<K>>::Error: Into<HttpError>,
|
||||
HeaderValue: TryFrom<V>,
|
||||
<HeaderValue as TryFrom<V>>::Error: Into<HttpError>,
|
||||
{
|
||||
self.map(|builder| builder.header(key, value))
|
||||
}
|
||||
|
||||
pub fn bearer_auth<T>(self, token: T) -> Self
|
||||
where
|
||||
T: Display,
|
||||
{
|
||||
self.map(|builder| builder.bearer_auth(token))
|
||||
}
|
||||
|
||||
pub fn json<T>(self, value: &T) -> Self
|
||||
where
|
||||
T: ?Sized + Serialize,
|
||||
{
|
||||
self.map(|builder| builder.json(value))
|
||||
}
|
||||
|
||||
pub async fn send(self) -> Result<Response, reqwest::Error> {
|
||||
match self.builder.send().await {
|
||||
Ok(response) => {
|
||||
let request_ids = Self::extract_request_ids(&response);
|
||||
tracing::debug!(
|
||||
method = %self.method,
|
||||
url = %self.url,
|
||||
status = %response.status(),
|
||||
request_ids = ?request_ids,
|
||||
version = ?response.version(),
|
||||
"Request completed"
|
||||
);
|
||||
|
||||
Ok(response)
|
||||
}
|
||||
Err(error) => {
|
||||
let status = error.status();
|
||||
tracing::debug!(
|
||||
method = %self.method,
|
||||
url = %self.url,
|
||||
status = status.map(|s| s.as_u16()),
|
||||
error = %error,
|
||||
"Request failed"
|
||||
);
|
||||
Err(error)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn extract_request_ids(response: &Response) -> HashMap<String, String> {
|
||||
["cf-ray", "x-request-id", "x-oai-request-id"]
|
||||
.iter()
|
||||
.filter_map(|&name| {
|
||||
let header_name = HeaderName::from_static(name);
|
||||
let value = response.headers().get(header_name)?;
|
||||
let value = value.to_str().ok()?.to_owned();
|
||||
Some((name.to_owned(), value))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Originator {
|
||||
pub value: String,
|
||||
@@ -256,8 +124,8 @@ fn sanitize_user_agent(candidate: String, fallback: &str) -> String {
|
||||
}
|
||||
}
|
||||
|
||||
/// Create an HTTP client with default `originator` and `User-Agent` headers set.
|
||||
pub fn create_client() -> CodexHttpClient {
|
||||
/// Create a reqwest client with default `originator` and `User-Agent` headers set.
|
||||
pub fn create_client() -> reqwest::Client {
|
||||
use reqwest::header::HeaderMap;
|
||||
|
||||
let mut headers = HeaderMap::new();
|
||||
@@ -272,8 +140,7 @@ pub fn create_client() -> CodexHttpClient {
|
||||
builder = builder.no_proxy();
|
||||
}
|
||||
|
||||
let inner = builder.build().unwrap_or_else(|_| reqwest::Client::new());
|
||||
CodexHttpClient::new(inner)
|
||||
builder.build().unwrap_or_else(|_| reqwest::Client::new())
|
||||
}
|
||||
|
||||
fn is_sandboxed() -> bool {
|
||||
|
||||
@@ -158,6 +158,9 @@ pub enum CodexErr {
|
||||
|
||||
#[error("{0}")]
|
||||
EnvVar(EnvVarError),
|
||||
|
||||
#[error("invalid input: {0}")]
|
||||
InvalidInput(String),
|
||||
}
|
||||
|
||||
impl From<CancelErr> for CodexErr {
|
||||
|
||||
@@ -20,7 +20,7 @@ pub mod config_edit;
|
||||
pub mod config_loader;
|
||||
pub mod config_profile;
|
||||
pub mod config_types;
|
||||
mod context_manager;
|
||||
mod conversation_history;
|
||||
pub mod custom_prompts;
|
||||
mod environment_context;
|
||||
pub mod error;
|
||||
|
||||
@@ -49,7 +49,7 @@ const MCP_TOOL_NAME_DELIMITER: &str = "__";
|
||||
const MAX_TOOL_NAME_LENGTH: usize = 64;
|
||||
|
||||
/// Default timeout for initializing MCP server & initially listing tools.
|
||||
pub const DEFAULT_STARTUP_TIMEOUT: Duration = Duration::from_secs(10);
|
||||
const DEFAULT_STARTUP_TIMEOUT: Duration = Duration::from_secs(10);
|
||||
|
||||
/// Default timeout for individual tool calls.
|
||||
const DEFAULT_TOOL_TIMEOUT: Duration = Duration::from_secs(60);
|
||||
|
||||
@@ -6,8 +6,6 @@
|
||||
//! key. These override or extend the defaults at runtime.
|
||||
|
||||
use crate::CodexAuth;
|
||||
use crate::default_client::CodexHttpClient;
|
||||
use crate::default_client::CodexRequestBuilder;
|
||||
use codex_app_server_protocol::AuthMode;
|
||||
use serde::Deserialize;
|
||||
use serde::Serialize;
|
||||
@@ -97,7 +95,7 @@ pub struct ModelProviderInfo {
|
||||
|
||||
impl ModelProviderInfo {
|
||||
/// Construct a `POST` RequestBuilder for the given URL using the provided
|
||||
/// [`CodexHttpClient`] applying:
|
||||
/// reqwest Client applying:
|
||||
/// • provider-specific headers (static + env based)
|
||||
/// • Bearer auth header when an API key is available.
|
||||
/// • Auth token for OAuth.
|
||||
@@ -106,9 +104,9 @@ impl ModelProviderInfo {
|
||||
/// one produced by [`ModelProviderInfo::api_key`].
|
||||
pub async fn create_request_builder<'a>(
|
||||
&'a self,
|
||||
client: &'a CodexHttpClient,
|
||||
client: &'a reqwest::Client,
|
||||
auth: &Option<CodexAuth>,
|
||||
) -> crate::error::Result<CodexRequestBuilder> {
|
||||
) -> crate::error::Result<reqwest::RequestBuilder> {
|
||||
let effective_auth = if let Some(secret_key) = &self.experimental_bearer_token {
|
||||
Some(CodexAuth::from_api_key(secret_key))
|
||||
} else {
|
||||
@@ -189,9 +187,9 @@ impl ModelProviderInfo {
|
||||
}
|
||||
|
||||
/// Apply provider-specific HTTP headers (both static and environment-based)
|
||||
/// onto an existing [`CodexRequestBuilder`] and return the updated
|
||||
/// onto an existing `reqwest::RequestBuilder` and return the updated
|
||||
/// builder.
|
||||
fn apply_http_headers(&self, mut builder: CodexRequestBuilder) -> CodexRequestBuilder {
|
||||
fn apply_http_headers(&self, mut builder: reqwest::RequestBuilder) -> reqwest::RequestBuilder {
|
||||
if let Some(extra) = &self.http_headers {
|
||||
for (k, v) in extra {
|
||||
builder = builder.header(k, v);
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
use crate::codex::Session;
|
||||
use crate::context_manager::ContextManager;
|
||||
use crate::conversation_history::ConversationHistory;
|
||||
use crate::error::Result as CodexResult;
|
||||
use codex_protocol::models::FunctionCallOutputPayload;
|
||||
use codex_protocol::models::ResponseInputItem;
|
||||
use codex_protocol::models::ResponseItem;
|
||||
@@ -11,9 +12,9 @@ use tracing::warn;
|
||||
pub(crate) async fn process_items(
|
||||
processed_items: Vec<crate::codex::ProcessedResponseItem>,
|
||||
is_review_mode: bool,
|
||||
review_thread_history: &mut ContextManager,
|
||||
review_thread_history: &mut ConversationHistory,
|
||||
sess: &Session,
|
||||
) -> (Vec<ResponseInputItem>, Vec<ResponseItem>) {
|
||||
) -> CodexResult<(Vec<ResponseInputItem>, Vec<ResponseItem>)> {
|
||||
let mut items_to_record_in_conversation_history = Vec::<ResponseItem>::new();
|
||||
let mut responses = Vec::<ResponseInputItem>::new();
|
||||
for processed_response_item in processed_items {
|
||||
@@ -102,11 +103,11 @@ pub(crate) async fn process_items(
|
||||
// Only attempt to take the lock if there is something to record.
|
||||
if !items_to_record_in_conversation_history.is_empty() {
|
||||
if is_review_mode {
|
||||
review_thread_history.record_items(items_to_record_in_conversation_history.iter());
|
||||
review_thread_history.record_items(items_to_record_in_conversation_history.iter())?;
|
||||
} else {
|
||||
sess.record_conversation_items(&items_to_record_in_conversation_history)
|
||||
.await;
|
||||
.await?;
|
||||
}
|
||||
}
|
||||
(responses, items_to_record_in_conversation_history)
|
||||
Ok((responses, items_to_record_in_conversation_history))
|
||||
}
|
||||
|
||||
@@ -3,7 +3,8 @@
|
||||
use codex_protocol::models::ResponseItem;
|
||||
|
||||
use crate::codex::SessionConfiguration;
|
||||
use crate::context_manager::ContextManager;
|
||||
use crate::conversation_history::ConversationHistory;
|
||||
use crate::error::CodexErr;
|
||||
use crate::protocol::RateLimitSnapshot;
|
||||
use crate::protocol::TokenUsage;
|
||||
use crate::protocol::TokenUsageInfo;
|
||||
@@ -11,7 +12,7 @@ use crate::protocol::TokenUsageInfo;
|
||||
/// Persistent, session-scoped state previously stored directly on `Session`.
|
||||
pub(crate) struct SessionState {
|
||||
pub(crate) session_configuration: SessionConfiguration,
|
||||
pub(crate) history: ContextManager,
|
||||
pub(crate) history: ConversationHistory,
|
||||
pub(crate) latest_rate_limits: Option<RateLimitSnapshot>,
|
||||
}
|
||||
|
||||
@@ -20,25 +21,26 @@ impl SessionState {
|
||||
pub(crate) fn new(session_configuration: SessionConfiguration) -> Self {
|
||||
Self {
|
||||
session_configuration,
|
||||
history: ContextManager::new(),
|
||||
history: ConversationHistory::new(),
|
||||
latest_rate_limits: None,
|
||||
}
|
||||
}
|
||||
|
||||
// History helpers
|
||||
pub(crate) fn record_items<I>(&mut self, items: I)
|
||||
pub(crate) fn record_items<I>(&mut self, items: I) -> Result<(), CodexErr>
|
||||
where
|
||||
I: IntoIterator,
|
||||
I::Item: std::ops::Deref<Target = ResponseItem>,
|
||||
{
|
||||
self.history.record_items(items)
|
||||
self.history.record_items(items)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn history_snapshot(&mut self) -> Vec<ResponseItem> {
|
||||
self.history.get_history()
|
||||
}
|
||||
|
||||
pub(crate) fn clone_history(&self) -> ContextManager {
|
||||
pub(crate) fn clone_history(&self) -> ConversationHistory {
|
||||
self.history.clone()
|
||||
}
|
||||
|
||||
@@ -66,7 +68,14 @@ impl SessionState {
|
||||
pub(crate) fn token_info_and_rate_limits(
|
||||
&self,
|
||||
) -> (Option<TokenUsageInfo>, Option<RateLimitSnapshot>) {
|
||||
(self.token_info(), self.latest_rate_limits.clone())
|
||||
let info = self.token_info().and_then(|info| {
|
||||
if info.total_token_usage.is_zero() && info.last_token_usage.is_zero() {
|
||||
None
|
||||
} else {
|
||||
Some(info)
|
||||
}
|
||||
});
|
||||
(info, self.latest_rate_limits.clone())
|
||||
}
|
||||
|
||||
pub(crate) fn set_token_usage_full(&mut self, context_window: i64) {
|
||||
|
||||
@@ -5,6 +5,7 @@ use tokio_util::sync::CancellationToken;
|
||||
|
||||
use crate::codex::TurnContext;
|
||||
use crate::codex::compact;
|
||||
use crate::error::Result as CodexResult;
|
||||
use crate::state::TaskKind;
|
||||
use codex_protocol::user_input::UserInput;
|
||||
|
||||
@@ -26,7 +27,7 @@ impl SessionTask for CompactTask {
|
||||
ctx: Arc<TurnContext>,
|
||||
input: Vec<UserInput>,
|
||||
_cancellation_token: CancellationToken,
|
||||
) -> Option<String> {
|
||||
) -> CodexResult<Option<String>> {
|
||||
compact::run_compact_task(session.clone_session(), ctx, input).await
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,6 +15,8 @@ use tracing::warn;
|
||||
|
||||
use crate::codex::Session;
|
||||
use crate::codex::TurnContext;
|
||||
use crate::error::Result as CodexResult;
|
||||
use crate::protocol::ErrorEvent;
|
||||
use crate::protocol::EventMsg;
|
||||
use crate::protocol::TaskCompleteEvent;
|
||||
use crate::protocol::TurnAbortReason;
|
||||
@@ -56,7 +58,7 @@ pub(crate) trait SessionTask: Send + Sync + 'static {
|
||||
ctx: Arc<TurnContext>,
|
||||
input: Vec<UserInput>,
|
||||
cancellation_token: CancellationToken,
|
||||
) -> Option<String>;
|
||||
) -> CodexResult<Option<String>>;
|
||||
|
||||
async fn abort(&self, session: Arc<SessionTaskContext>, ctx: Arc<TurnContext>) {
|
||||
let _ = (session, ctx);
|
||||
@@ -86,7 +88,7 @@ impl Session {
|
||||
let task_cancellation_token = cancellation_token.child_token();
|
||||
tokio::spawn(async move {
|
||||
let ctx_for_finish = Arc::clone(&ctx);
|
||||
let last_agent_message = task_for_run
|
||||
let run_result = task_for_run
|
||||
.run(
|
||||
Arc::clone(&session_ctx),
|
||||
ctx,
|
||||
@@ -98,8 +100,21 @@ impl Session {
|
||||
if !task_cancellation_token.is_cancelled() {
|
||||
// Emit completion uniformly from spawn site so all tasks share the same lifecycle.
|
||||
let sess = session_ctx.clone_session();
|
||||
sess.on_task_finished(ctx_for_finish, last_agent_message)
|
||||
.await;
|
||||
match run_result {
|
||||
Ok(last_agent_message) => {
|
||||
sess.on_task_finished(ctx_for_finish, last_agent_message)
|
||||
.await;
|
||||
}
|
||||
Err(err) => {
|
||||
let message = err.to_string();
|
||||
sess.send_event(
|
||||
ctx_for_finish.as_ref(),
|
||||
EventMsg::Error(ErrorEvent { message }),
|
||||
)
|
||||
.await;
|
||||
sess.on_task_finished(ctx_for_finish, None).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
done_clone.notify_waiters();
|
||||
})
|
||||
|
||||
@@ -5,6 +5,7 @@ use tokio_util::sync::CancellationToken;
|
||||
|
||||
use crate::codex::TurnContext;
|
||||
use crate::codex::run_task;
|
||||
use crate::error::Result as CodexResult;
|
||||
use crate::state::TaskKind;
|
||||
use codex_protocol::user_input::UserInput;
|
||||
|
||||
@@ -26,7 +27,7 @@ impl SessionTask for RegularTask {
|
||||
ctx: Arc<TurnContext>,
|
||||
input: Vec<UserInput>,
|
||||
cancellation_token: CancellationToken,
|
||||
) -> Option<String> {
|
||||
) -> CodexResult<Option<String>> {
|
||||
let sess = session.clone_session();
|
||||
run_task(sess, ctx, input, TaskKind::Regular, cancellation_token).await
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ use tokio_util::sync::CancellationToken;
|
||||
use crate::codex::TurnContext;
|
||||
use crate::codex::exit_review_mode;
|
||||
use crate::codex::run_task;
|
||||
use crate::error::Result as CodexResult;
|
||||
use crate::state::TaskKind;
|
||||
use codex_protocol::user_input::UserInput;
|
||||
|
||||
@@ -27,12 +28,12 @@ impl SessionTask for ReviewTask {
|
||||
ctx: Arc<TurnContext>,
|
||||
input: Vec<UserInput>,
|
||||
cancellation_token: CancellationToken,
|
||||
) -> Option<String> {
|
||||
) -> CodexResult<Option<String>> {
|
||||
let sess = session.clone_session();
|
||||
run_task(sess, ctx, input, TaskKind::Review, cancellation_token).await
|
||||
}
|
||||
|
||||
async fn abort(&self, session: Arc<SessionTaskContext>, ctx: Arc<TurnContext>) {
|
||||
exit_review_mode(session.clone_session(), ctx, None).await;
|
||||
let _ = exit_review_mode(session.clone_session(), ctx, None).await;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,11 +1,15 @@
|
||||
use crate::codex::Session;
|
||||
use crate::codex::TurnContext;
|
||||
use crate::tools::TELEMETRY_PREVIEW_MAX_BYTES;
|
||||
use crate::tools::TELEMETRY_PREVIEW_MAX_LINES;
|
||||
use crate::tools::TELEMETRY_PREVIEW_TRUNCATION_NOTICE;
|
||||
use crate::turn_diff_tracker::TurnDiffTracker;
|
||||
use codex_otel::otel_event_manager::OtelEventManager;
|
||||
use codex_protocol::models::FunctionCallOutputPayload;
|
||||
use codex_protocol::models::ResponseInputItem;
|
||||
use codex_protocol::models::ShellToolCallParams;
|
||||
use codex_protocol::protocol::FileChange;
|
||||
use codex_utils_string::take_bytes_at_char_boundary;
|
||||
use mcp_types::CallToolResult;
|
||||
use std::borrow::Cow;
|
||||
use std::collections::HashMap;
|
||||
@@ -72,7 +76,7 @@ pub enum ToolOutput {
|
||||
impl ToolOutput {
|
||||
pub fn log_preview(&self) -> String {
|
||||
match self {
|
||||
ToolOutput::Function { content, .. } => content.clone(),
|
||||
ToolOutput::Function { content, .. } => telemetry_preview(content),
|
||||
ToolOutput::Mcp { result } => format!("{result:?}"),
|
||||
}
|
||||
}
|
||||
@@ -107,6 +111,46 @@ impl ToolOutput {
|
||||
}
|
||||
}
|
||||
|
||||
fn telemetry_preview(content: &str) -> String {
|
||||
let truncated_slice = take_bytes_at_char_boundary(content, TELEMETRY_PREVIEW_MAX_BYTES);
|
||||
let truncated_by_bytes = truncated_slice.len() < content.len();
|
||||
|
||||
let mut preview = String::new();
|
||||
let mut lines_iter = truncated_slice.lines();
|
||||
for idx in 0..TELEMETRY_PREVIEW_MAX_LINES {
|
||||
match lines_iter.next() {
|
||||
Some(line) => {
|
||||
if idx > 0 {
|
||||
preview.push('\n');
|
||||
}
|
||||
preview.push_str(line);
|
||||
}
|
||||
None => break,
|
||||
}
|
||||
}
|
||||
let truncated_by_lines = lines_iter.next().is_some();
|
||||
|
||||
if !truncated_by_bytes && !truncated_by_lines {
|
||||
return content.to_string();
|
||||
}
|
||||
|
||||
if preview.len() < truncated_slice.len()
|
||||
&& truncated_slice
|
||||
.as_bytes()
|
||||
.get(preview.len())
|
||||
.is_some_and(|byte| *byte == b'\n')
|
||||
{
|
||||
preview.push('\n');
|
||||
}
|
||||
|
||||
if !preview.is_empty() && !preview.ends_with('\n') {
|
||||
preview.push('\n');
|
||||
}
|
||||
preview.push_str(TELEMETRY_PREVIEW_TRUNCATION_NOTICE);
|
||||
|
||||
preview
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -152,6 +196,38 @@ mod tests {
|
||||
other => panic!("expected FunctionCallOutput, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn telemetry_preview_returns_original_within_limits() {
|
||||
let content = "short output";
|
||||
assert_eq!(telemetry_preview(content), content);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn telemetry_preview_truncates_by_bytes() {
|
||||
let content = "x".repeat(TELEMETRY_PREVIEW_MAX_BYTES + 8);
|
||||
let preview = telemetry_preview(&content);
|
||||
|
||||
assert!(preview.contains(TELEMETRY_PREVIEW_TRUNCATION_NOTICE));
|
||||
assert!(
|
||||
preview.len()
|
||||
<= TELEMETRY_PREVIEW_MAX_BYTES + TELEMETRY_PREVIEW_TRUNCATION_NOTICE.len() + 1
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn telemetry_preview_truncates_by_lines() {
|
||||
let content = (0..(TELEMETRY_PREVIEW_MAX_LINES + 5))
|
||||
.map(|idx| format!("line {idx}"))
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n");
|
||||
|
||||
let preview = telemetry_preview(&content);
|
||||
let lines: Vec<&str> = preview.lines().collect();
|
||||
|
||||
assert!(lines.len() <= TELEMETRY_PREVIEW_MAX_LINES + 1);
|
||||
assert_eq!(lines.last(), Some(&TELEMETRY_PREVIEW_TRUNCATION_NOTICE));
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
|
||||
@@ -22,6 +22,12 @@ pub(crate) const MODEL_FORMAT_HEAD_LINES: usize = MODEL_FORMAT_MAX_LINES / 2;
|
||||
pub(crate) const MODEL_FORMAT_TAIL_LINES: usize = MODEL_FORMAT_MAX_LINES - MODEL_FORMAT_HEAD_LINES; // 128
|
||||
pub(crate) const MODEL_FORMAT_HEAD_BYTES: usize = MODEL_FORMAT_MAX_BYTES / 2;
|
||||
|
||||
// Telemetry preview limits: keep log events smaller than model budgets.
|
||||
pub(crate) const TELEMETRY_PREVIEW_MAX_BYTES: usize = 2 * 1024; // 2 KiB
|
||||
pub(crate) const TELEMETRY_PREVIEW_MAX_LINES: usize = 64; // lines
|
||||
pub(crate) const TELEMETRY_PREVIEW_TRUNCATION_NOTICE: &str =
|
||||
"[... telemetry preview truncated ...]";
|
||||
|
||||
/// Format the combined exec output for sending back to the model.
|
||||
/// Includes exit code and duration metadata; truncates large bodies safely.
|
||||
pub fn format_exec_output_for_model(exec_output: &ExecToolCallOutput) -> String {
|
||||
|
||||
@@ -279,11 +279,6 @@ async fn auto_compact_runs_after_token_limit_hit() {
|
||||
ev_completed_with_tokens("r2", 330_000),
|
||||
]);
|
||||
|
||||
let sse3 = sse(vec![
|
||||
ev_assistant_message("m3", AUTO_SUMMARY_TEXT),
|
||||
ev_completed_with_tokens("r3", 200),
|
||||
]);
|
||||
|
||||
let first_matcher = |req: &wiremock::Request| {
|
||||
let body = std::str::from_utf8(&req.body).unwrap_or("");
|
||||
body.contains(FIRST_AUTO_MSG)
|
||||
@@ -300,12 +295,6 @@ async fn auto_compact_runs_after_token_limit_hit() {
|
||||
};
|
||||
mount_sse_once_match(&server, second_matcher, sse2).await;
|
||||
|
||||
let third_matcher = |req: &wiremock::Request| {
|
||||
let body = std::str::from_utf8(&req.body).unwrap_or("");
|
||||
body.contains("You have exceeded the maximum number of tokens")
|
||||
};
|
||||
mount_sse_once_match(&server, third_matcher, sse3).await;
|
||||
|
||||
let model_provider = ModelProviderInfo {
|
||||
base_url: Some(format!("{}/v1", server.uri())),
|
||||
..built_in_model_providers()["openai"].clone()
|
||||
@@ -342,69 +331,28 @@ async fn auto_compact_runs_after_token_limit_hit() {
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let error_event = wait_for_event(&codex, |ev| matches!(ev, EventMsg::Error(_))).await;
|
||||
let EventMsg::Error(error_event) = error_event else {
|
||||
unreachable!("wait_for_event returned unexpected payload");
|
||||
};
|
||||
assert_eq!(error_event.message, "invalid input: input too large");
|
||||
|
||||
wait_for_event(&codex, |ev| matches!(ev, EventMsg::TaskComplete(_))).await;
|
||||
// wait_for_event(&codex, |ev| matches!(ev, EventMsg::TaskComplete(_))).await;
|
||||
|
||||
let requests = server.received_requests().await.unwrap();
|
||||
assert!(
|
||||
requests.len() >= 3,
|
||||
"auto compact should add at least a third request, got {}",
|
||||
requests.len()
|
||||
assert_eq!(
|
||||
requests.len(),
|
||||
2,
|
||||
"auto compact should reject oversize prompts before issuing another request"
|
||||
);
|
||||
let is_auto_compact = |req: &wiremock::Request| {
|
||||
let saw_compact_prompt = requests.iter().any(|req| {
|
||||
std::str::from_utf8(&req.body)
|
||||
.unwrap_or("")
|
||||
.contains("You have exceeded the maximum number of tokens")
|
||||
};
|
||||
let auto_compact_count = requests.iter().filter(|req| is_auto_compact(req)).count();
|
||||
assert_eq!(
|
||||
auto_compact_count, 1,
|
||||
"expected exactly one auto compact request"
|
||||
);
|
||||
let auto_compact_index = requests
|
||||
.iter()
|
||||
.enumerate()
|
||||
.find_map(|(idx, req)| is_auto_compact(req).then_some(idx))
|
||||
.expect("auto compact request missing");
|
||||
assert_eq!(
|
||||
auto_compact_index, 2,
|
||||
"auto compact should add a third request"
|
||||
);
|
||||
|
||||
let body_first = requests[0].body_json::<serde_json::Value>().unwrap();
|
||||
let body3 = requests[auto_compact_index]
|
||||
.body_json::<serde_json::Value>()
|
||||
.unwrap();
|
||||
let instructions = body3
|
||||
.get("instructions")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or_default();
|
||||
let baseline_instructions = body_first
|
||||
.get("instructions")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or_default()
|
||||
.to_string();
|
||||
assert_eq!(
|
||||
instructions, baseline_instructions,
|
||||
"auto compact should keep the standard developer instructions",
|
||||
);
|
||||
|
||||
let input3 = body3.get("input").and_then(|v| v.as_array()).unwrap();
|
||||
let last3 = input3
|
||||
.last()
|
||||
.expect("auto compact request should append a user message");
|
||||
assert_eq!(last3.get("type").and_then(|v| v.as_str()), Some("message"));
|
||||
assert_eq!(last3.get("role").and_then(|v| v.as_str()), Some("user"));
|
||||
let last_text = last3
|
||||
.get("content")
|
||||
.and_then(|v| v.as_array())
|
||||
.and_then(|items| items.first())
|
||||
.and_then(|item| item.get("text"))
|
||||
.and_then(|text| text.as_str())
|
||||
.unwrap_or_default();
|
||||
assert_eq!(
|
||||
last_text, SUMMARIZATION_PROMPT,
|
||||
"auto compact should send the summarization prompt as a user message",
|
||||
});
|
||||
assert!(
|
||||
!saw_compact_prompt,
|
||||
"no auto compact request should be sent when the summarization prompt exceeds the limit"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -869,7 +817,7 @@ async fn auto_compact_triggers_after_function_call_over_95_percent_usage() {
|
||||
|
||||
let server = start_mock_server().await;
|
||||
|
||||
let context_window = 100;
|
||||
let context_window = 20_000;
|
||||
let limit = context_window * 90 / 100;
|
||||
let over_limit_tokens = context_window * 95 / 100 + 1;
|
||||
|
||||
|
||||
69
codex-rs/core/tests/suite/input_validation.rs
Normal file
69
codex-rs/core/tests/suite/input_validation.rs
Normal file
@@ -0,0 +1,69 @@
|
||||
use codex_core::protocol::EventMsg;
|
||||
use codex_core::protocol::Op;
|
||||
use codex_protocol::user_input::UserInput;
|
||||
use core_test_support::responses;
|
||||
use core_test_support::responses::ev_assistant_message;
|
||||
use core_test_support::responses::ev_completed;
|
||||
use core_test_support::responses::ev_response_created;
|
||||
use core_test_support::responses::sse;
|
||||
use core_test_support::responses::start_mock_server;
|
||||
use core_test_support::test_codex::test_codex;
|
||||
use core_test_support::wait_for_event_with_timeout;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use wiremock::matchers::any;
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn input_validation_should_fail_for_too_large_input() {
|
||||
let server = start_mock_server().await;
|
||||
|
||||
let fixture = test_codex().build(&server).await.unwrap();
|
||||
let codex = Arc::clone(&fixture.codex);
|
||||
|
||||
// First: normal message with a mocked assistant response
|
||||
let first_response = sse(vec![
|
||||
ev_response_created("resp-1"),
|
||||
ev_assistant_message("msg-1", "ok"),
|
||||
ev_completed("resp-1"),
|
||||
]);
|
||||
responses::mount_sse_once_match(&server, any(), first_response).await;
|
||||
|
||||
codex
|
||||
.submit(Op::UserInput {
|
||||
items: vec![UserInput::Text {
|
||||
text: "hello world".into(),
|
||||
}],
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Wait for the normal turn to complete before sending the oversized input
|
||||
let turn_timeout = Duration::from_secs(1);
|
||||
wait_for_event_with_timeout(
|
||||
&codex,
|
||||
|ev| matches!(ev, EventMsg::TaskComplete(_)),
|
||||
turn_timeout,
|
||||
)
|
||||
.await;
|
||||
|
||||
// Then: 300k-token message should trigger validation error
|
||||
let wait_timeout = Duration::from_millis(100);
|
||||
let input_300_tokens = "token ".repeat(300_000);
|
||||
|
||||
codex
|
||||
.submit(Op::UserInput {
|
||||
items: vec![UserInput::Text {
|
||||
text: input_300_tokens,
|
||||
}],
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let error_event =
|
||||
wait_for_event_with_timeout(&codex, |ev| matches!(ev, EventMsg::Error(_)), wait_timeout)
|
||||
.await;
|
||||
let EventMsg::Error(error_event) = error_event else {
|
||||
unreachable!("wait_for_event_with_timeout returned unexpected payload");
|
||||
};
|
||||
assert_eq!(error_event.message, "invalid input: input too large");
|
||||
}
|
||||
@@ -13,6 +13,7 @@ mod compact_resume_fork;
|
||||
mod exec;
|
||||
mod fork_conversation;
|
||||
mod grep_files;
|
||||
mod input_validation;
|
||||
mod items;
|
||||
mod json_result;
|
||||
mod list_dir;
|
||||
|
||||
@@ -156,12 +156,6 @@ async fn unified_exec_emits_exec_command_end_event() -> Result<()> {
|
||||
"cmd": "/bin/echo END-EVENT".to_string(),
|
||||
"yield_time_ms": 250,
|
||||
});
|
||||
let poll_call_id = "uexec-end-event-poll";
|
||||
let poll_args = json!({
|
||||
"chars": "",
|
||||
"session_id": 0,
|
||||
"yield_time_ms": 250,
|
||||
});
|
||||
|
||||
let responses = vec![
|
||||
sse(vec![
|
||||
@@ -171,17 +165,8 @@ async fn unified_exec_emits_exec_command_end_event() -> Result<()> {
|
||||
]),
|
||||
sse(vec![
|
||||
ev_response_created("resp-2"),
|
||||
ev_function_call(
|
||||
poll_call_id,
|
||||
"write_stdin",
|
||||
&serde_json::to_string(&poll_args)?,
|
||||
),
|
||||
ev_completed("resp-2"),
|
||||
]),
|
||||
sse(vec![
|
||||
ev_response_created("resp-3"),
|
||||
ev_assistant_message("msg-1", "finished"),
|
||||
ev_completed("resp-3"),
|
||||
ev_completed("resp-2"),
|
||||
]),
|
||||
];
|
||||
mount_sse_sequence(&server, responses).await;
|
||||
|
||||
@@ -12,7 +12,7 @@ use anyhow::anyhow;
|
||||
use codex_protocol::ConversationId;
|
||||
use tracing_subscriber::fmt::writer::MakeWriter;
|
||||
|
||||
const DEFAULT_MAX_BYTES: usize = 4 * 1024 * 1024; // 4 MiB
|
||||
const DEFAULT_MAX_BYTES: usize = 2 * 1024 * 1024; // 2 MiB
|
||||
const SENTRY_DSN: &str =
|
||||
"https://ae32ed50620d7a7792c1ce5df38b3e3e@o33249.ingest.us.sentry.io/4510195390611458";
|
||||
const UPLOAD_TIMEOUT_SECS: u64 = 10;
|
||||
|
||||
@@ -575,9 +575,11 @@ pub struct TokenUsage {
|
||||
pub total_tokens: i64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, TS)]
|
||||
#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, TS, Default)]
|
||||
pub struct TokenUsageInfo {
|
||||
/// The total token usage for the session. accumulated from all turns.
|
||||
pub total_token_usage: TokenUsage,
|
||||
/// The token usage for the last turn. Received from the API. It's total tokens is the whole window size.
|
||||
pub last_token_usage: TokenUsage,
|
||||
#[ts(type = "number | null")]
|
||||
pub model_context_window: Option<i64>,
|
||||
|
||||
@@ -1631,7 +1631,6 @@ impl ChatWidget {
|
||||
context_usage,
|
||||
&self.conversation_id,
|
||||
self.rate_limit_snapshot.as_ref(),
|
||||
Local::now(),
|
||||
));
|
||||
}
|
||||
|
||||
|
||||
@@ -3,8 +3,6 @@ use crate::history_cell::HistoryCell;
|
||||
use crate::history_cell::PlainHistoryCell;
|
||||
use crate::history_cell::with_border_with_inner_width;
|
||||
use crate::version::CODEX_CLI_VERSION;
|
||||
use chrono::DateTime;
|
||||
use chrono::Local;
|
||||
use codex_common::create_config_summary_entries;
|
||||
use codex_core::config::Config;
|
||||
use codex_core::protocol::SandboxPolicy;
|
||||
@@ -27,7 +25,6 @@ use super::helpers::format_directory_display;
|
||||
use super::helpers::format_tokens_compact;
|
||||
use super::rate_limits::RateLimitSnapshotDisplay;
|
||||
use super::rate_limits::StatusRateLimitData;
|
||||
use super::rate_limits::StatusRateLimitRow;
|
||||
use super::rate_limits::compose_rate_limit_data;
|
||||
use super::rate_limits::format_status_limit_summary;
|
||||
use super::rate_limits::render_status_limit_progress_bar;
|
||||
@@ -67,17 +64,9 @@ pub(crate) fn new_status_output(
|
||||
context_usage: Option<&TokenUsage>,
|
||||
session_id: &Option<ConversationId>,
|
||||
rate_limits: Option<&RateLimitSnapshotDisplay>,
|
||||
now: DateTime<Local>,
|
||||
) -> CompositeHistoryCell {
|
||||
let command = PlainHistoryCell::new(vec!["/status".magenta().into()]);
|
||||
let card = StatusHistoryCell::new(
|
||||
config,
|
||||
total_usage,
|
||||
context_usage,
|
||||
session_id,
|
||||
rate_limits,
|
||||
now,
|
||||
);
|
||||
let card = StatusHistoryCell::new(config, total_usage, context_usage, session_id, rate_limits);
|
||||
|
||||
CompositeHistoryCell::new(vec![Box::new(command), Box::new(card)])
|
||||
}
|
||||
@@ -89,7 +78,6 @@ impl StatusHistoryCell {
|
||||
context_usage: Option<&TokenUsage>,
|
||||
session_id: &Option<ConversationId>,
|
||||
rate_limits: Option<&RateLimitSnapshotDisplay>,
|
||||
now: DateTime<Local>,
|
||||
) -> Self {
|
||||
let config_entries = create_config_summary_entries(config);
|
||||
let (model_name, model_details) = compose_model_display(config, &config_entries);
|
||||
@@ -120,7 +108,7 @@ impl StatusHistoryCell {
|
||||
output: total_usage.output_tokens,
|
||||
context_window,
|
||||
};
|
||||
let rate_limits = compose_rate_limit_data(rate_limits, now);
|
||||
let rate_limits = compose_rate_limit_data(rate_limits);
|
||||
|
||||
Self {
|
||||
model_name,
|
||||
@@ -183,66 +171,47 @@ impl StatusHistoryCell {
|
||||
];
|
||||
}
|
||||
|
||||
self.rate_limit_row_lines(rows_data, available_inner_width, formatter)
|
||||
}
|
||||
StatusRateLimitData::Stale(rows_data) => {
|
||||
let mut lines =
|
||||
self.rate_limit_row_lines(rows_data, available_inner_width, formatter);
|
||||
lines.push(formatter.line(
|
||||
"Warning",
|
||||
vec![Span::from("limits may be stale - start new turn to refresh.").dim()],
|
||||
));
|
||||
let mut lines = Vec::with_capacity(rows_data.len() * 2);
|
||||
|
||||
for row in rows_data {
|
||||
let value_spans = vec![
|
||||
Span::from(render_status_limit_progress_bar(row.percent_used)),
|
||||
Span::from(" "),
|
||||
Span::from(format_status_limit_summary(row.percent_used)),
|
||||
];
|
||||
let base_spans = formatter.full_spans(row.label.as_str(), value_spans);
|
||||
let base_line = Line::from(base_spans.clone());
|
||||
|
||||
if let Some(resets_at) = row.resets_at.as_ref() {
|
||||
let resets_span = Span::from(format!("(resets {resets_at})")).dim();
|
||||
let mut inline_spans = base_spans.clone();
|
||||
inline_spans.push(Span::from(" ").dim());
|
||||
inline_spans.push(resets_span.clone());
|
||||
|
||||
if line_display_width(&Line::from(inline_spans.clone()))
|
||||
<= available_inner_width
|
||||
{
|
||||
lines.push(Line::from(inline_spans));
|
||||
} else {
|
||||
lines.push(base_line);
|
||||
lines.push(formatter.continuation(vec![resets_span]));
|
||||
}
|
||||
} else {
|
||||
lines.push(base_line);
|
||||
}
|
||||
}
|
||||
|
||||
lines
|
||||
}
|
||||
StatusRateLimitData::Missing => {
|
||||
vec![formatter.line(
|
||||
"Limits",
|
||||
vec![
|
||||
Span::from("visit ").dim(),
|
||||
"chatgpt.com/codex/settings/usage".cyan().underlined(),
|
||||
],
|
||||
vec![Span::from("send a message to load usage data").dim()],
|
||||
)]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn rate_limit_row_lines(
|
||||
&self,
|
||||
rows: &[StatusRateLimitRow],
|
||||
available_inner_width: usize,
|
||||
formatter: &FieldFormatter,
|
||||
) -> Vec<Line<'static>> {
|
||||
let mut lines = Vec::with_capacity(rows.len().saturating_mul(2));
|
||||
|
||||
for row in rows {
|
||||
let value_spans = vec![
|
||||
Span::from(render_status_limit_progress_bar(row.percent_used)),
|
||||
Span::from(" "),
|
||||
Span::from(format_status_limit_summary(row.percent_used)),
|
||||
];
|
||||
let base_spans = formatter.full_spans(row.label.as_str(), value_spans);
|
||||
let base_line = Line::from(base_spans.clone());
|
||||
|
||||
if let Some(resets_at) = row.resets_at.as_ref() {
|
||||
let resets_span = Span::from(format!("(resets {resets_at})")).dim();
|
||||
let mut inline_spans = base_spans.clone();
|
||||
inline_spans.push(Span::from(" ").dim());
|
||||
inline_spans.push(resets_span.clone());
|
||||
|
||||
if line_display_width(&Line::from(inline_spans.clone())) <= available_inner_width {
|
||||
lines.push(Line::from(inline_spans));
|
||||
} else {
|
||||
lines.push(base_line);
|
||||
lines.push(formatter.continuation(vec![resets_span]));
|
||||
}
|
||||
} else {
|
||||
lines.push(base_line);
|
||||
}
|
||||
}
|
||||
|
||||
lines
|
||||
}
|
||||
|
||||
fn collect_rate_limit_labels(&self, seen: &mut BTreeSet<String>, labels: &mut Vec<String>) {
|
||||
match &self.rate_limits {
|
||||
StatusRateLimitData::Available(rows) => {
|
||||
@@ -254,12 +223,6 @@ impl StatusHistoryCell {
|
||||
}
|
||||
}
|
||||
}
|
||||
StatusRateLimitData::Stale(rows) => {
|
||||
for row in rows {
|
||||
push_label(labels, seen, row.label.as_str());
|
||||
}
|
||||
push_label(labels, seen, "Warning");
|
||||
}
|
||||
StatusRateLimitData::Missing => push_label(labels, seen, "Limits"),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,7 +2,6 @@ use crate::chatwidget::get_limits_duration;
|
||||
|
||||
use super::helpers::format_reset_timestamp;
|
||||
use chrono::DateTime;
|
||||
use chrono::Duration as ChronoDuration;
|
||||
use chrono::Local;
|
||||
use chrono::Utc;
|
||||
use codex_core::protocol::RateLimitSnapshot;
|
||||
@@ -22,12 +21,9 @@ pub(crate) struct StatusRateLimitRow {
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) enum StatusRateLimitData {
|
||||
Available(Vec<StatusRateLimitRow>),
|
||||
Stale(Vec<StatusRateLimitRow>),
|
||||
Missing,
|
||||
}
|
||||
|
||||
pub(crate) const RATE_LIMIT_STALE_THRESHOLD_MINUTES: i64 = 15;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) struct RateLimitWindowDisplay {
|
||||
pub used_percent: f64,
|
||||
@@ -53,7 +49,6 @@ impl RateLimitWindowDisplay {
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) struct RateLimitSnapshotDisplay {
|
||||
pub captured_at: DateTime<Local>,
|
||||
pub primary: Option<RateLimitWindowDisplay>,
|
||||
pub secondary: Option<RateLimitWindowDisplay>,
|
||||
}
|
||||
@@ -63,7 +58,6 @@ pub(crate) fn rate_limit_snapshot_display(
|
||||
captured_at: DateTime<Local>,
|
||||
) -> RateLimitSnapshotDisplay {
|
||||
RateLimitSnapshotDisplay {
|
||||
captured_at,
|
||||
primary: snapshot
|
||||
.primary
|
||||
.as_ref()
|
||||
@@ -77,7 +71,6 @@ pub(crate) fn rate_limit_snapshot_display(
|
||||
|
||||
pub(crate) fn compose_rate_limit_data(
|
||||
snapshot: Option<&RateLimitSnapshotDisplay>,
|
||||
now: DateTime<Local>,
|
||||
) -> StatusRateLimitData {
|
||||
match snapshot {
|
||||
Some(snapshot) => {
|
||||
@@ -109,13 +102,8 @@ pub(crate) fn compose_rate_limit_data(
|
||||
});
|
||||
}
|
||||
|
||||
let is_stale = now.signed_duration_since(snapshot.captured_at)
|
||||
> ChronoDuration::minutes(RATE_LIMIT_STALE_THRESHOLD_MINUTES);
|
||||
|
||||
if rows.is_empty() {
|
||||
StatusRateLimitData::Available(vec![])
|
||||
} else if is_stale {
|
||||
StatusRateLimitData::Stale(rows)
|
||||
} else {
|
||||
StatusRateLimitData::Available(rows)
|
||||
}
|
||||
|
||||
@@ -15,5 +15,5 @@ expression: sanitized
|
||||
│ │
|
||||
│ Token usage: 750 total (500 input + 250 output) │
|
||||
│ Context window: 100% left (750 used / 272K) │
|
||||
│ Limits: visit chatgpt.com/codex/settings/usage │
|
||||
│ Limits: send a message to load usage data │
|
||||
╰─────────────────────────────────────────────────────────────────╯
|
||||
|
||||
@@ -1,21 +0,0 @@
|
||||
---
|
||||
source: tui/src/status/tests.rs
|
||||
expression: sanitized
|
||||
---
|
||||
/status
|
||||
|
||||
╭─────────────────────────────────────────────────────────────────────╮
|
||||
│ >_ OpenAI Codex (v0.0.0) │
|
||||
│ │
|
||||
│ Model: gpt-5-codex (reasoning none, summaries auto) │
|
||||
│ Directory: [[workspace]] │
|
||||
│ Approval: on-request │
|
||||
│ Sandbox: read-only │
|
||||
│ Agents.md: <none> │
|
||||
│ │
|
||||
│ Token usage: 1.9K total (1K input + 900 output) │
|
||||
│ Context window: 100% left (2.1K used / 272K) │
|
||||
│ 5h limit: [███████████████░░░░░] 72% used (resets 03:14) │
|
||||
│ Weekly limit: [████████░░░░░░░░░░░░] 40% used (resets 03:34) │
|
||||
│ Warning: limits may be stale - start new turn to refresh. │
|
||||
╰─────────────────────────────────────────────────────────────────────╯
|
||||
@@ -111,14 +111,7 @@ fn status_snapshot_includes_reasoning_details() {
|
||||
};
|
||||
let rate_display = rate_limit_snapshot_display(&snapshot, captured_at);
|
||||
|
||||
let composite = new_status_output(
|
||||
&config,
|
||||
&usage,
|
||||
Some(&usage),
|
||||
&None,
|
||||
Some(&rate_display),
|
||||
captured_at,
|
||||
);
|
||||
let composite = new_status_output(&config, &usage, Some(&usage), &None, Some(&rate_display));
|
||||
let mut rendered_lines = render_lines(&composite.display_lines(80));
|
||||
if cfg!(windows) {
|
||||
for line in &mut rendered_lines {
|
||||
@@ -159,14 +152,7 @@ fn status_snapshot_includes_monthly_limit() {
|
||||
};
|
||||
let rate_display = rate_limit_snapshot_display(&snapshot, captured_at);
|
||||
|
||||
let composite = new_status_output(
|
||||
&config,
|
||||
&usage,
|
||||
Some(&usage),
|
||||
&None,
|
||||
Some(&rate_display),
|
||||
captured_at,
|
||||
);
|
||||
let composite = new_status_output(&config, &usage, Some(&usage), &None, Some(&rate_display));
|
||||
let mut rendered_lines = render_lines(&composite.display_lines(80));
|
||||
if cfg!(windows) {
|
||||
for line in &mut rendered_lines {
|
||||
@@ -192,12 +178,7 @@ fn status_card_token_usage_excludes_cached_tokens() {
|
||||
total_tokens: 2_100,
|
||||
};
|
||||
|
||||
let now = chrono::Local
|
||||
.with_ymd_and_hms(2024, 1, 1, 0, 0, 0)
|
||||
.single()
|
||||
.expect("timestamp");
|
||||
|
||||
let composite = new_status_output(&config, &usage, Some(&usage), &None, None, now);
|
||||
let composite = new_status_output(&config, &usage, Some(&usage), &None, None);
|
||||
let rendered = render_lines(&composite.display_lines(120));
|
||||
|
||||
assert!(
|
||||
@@ -238,14 +219,7 @@ fn status_snapshot_truncates_in_narrow_terminal() {
|
||||
};
|
||||
let rate_display = rate_limit_snapshot_display(&snapshot, captured_at);
|
||||
|
||||
let composite = new_status_output(
|
||||
&config,
|
||||
&usage,
|
||||
Some(&usage),
|
||||
&None,
|
||||
Some(&rate_display),
|
||||
captured_at,
|
||||
);
|
||||
let composite = new_status_output(&config, &usage, Some(&usage), &None, Some(&rate_display));
|
||||
let mut rendered_lines = render_lines(&composite.display_lines(46));
|
||||
if cfg!(windows) {
|
||||
for line in &mut rendered_lines {
|
||||
@@ -272,12 +246,7 @@ fn status_snapshot_shows_missing_limits_message() {
|
||||
total_tokens: 750,
|
||||
};
|
||||
|
||||
let now = chrono::Local
|
||||
.with_ymd_and_hms(2024, 2, 3, 4, 5, 6)
|
||||
.single()
|
||||
.expect("timestamp");
|
||||
|
||||
let composite = new_status_output(&config, &usage, Some(&usage), &None, None, now);
|
||||
let composite = new_status_output(&config, &usage, Some(&usage), &None, None);
|
||||
let mut rendered_lines = render_lines(&composite.display_lines(80));
|
||||
if cfg!(windows) {
|
||||
for line in &mut rendered_lines {
|
||||
@@ -313,66 +282,7 @@ fn status_snapshot_shows_empty_limits_message() {
|
||||
.expect("timestamp");
|
||||
let rate_display = rate_limit_snapshot_display(&snapshot, captured_at);
|
||||
|
||||
let composite = new_status_output(
|
||||
&config,
|
||||
&usage,
|
||||
Some(&usage),
|
||||
&None,
|
||||
Some(&rate_display),
|
||||
captured_at,
|
||||
);
|
||||
let mut rendered_lines = render_lines(&composite.display_lines(80));
|
||||
if cfg!(windows) {
|
||||
for line in &mut rendered_lines {
|
||||
*line = line.replace('\\', "/");
|
||||
}
|
||||
}
|
||||
let sanitized = sanitize_directory(rendered_lines).join("\n");
|
||||
assert_snapshot!(sanitized);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn status_snapshot_shows_stale_limits_message() {
|
||||
let temp_home = TempDir::new().expect("temp home");
|
||||
let mut config = test_config(&temp_home);
|
||||
config.model = "gpt-5-codex".to_string();
|
||||
config.cwd = PathBuf::from("/workspace/tests");
|
||||
|
||||
let usage = TokenUsage {
|
||||
input_tokens: 1_200,
|
||||
cached_input_tokens: 200,
|
||||
output_tokens: 900,
|
||||
reasoning_output_tokens: 150,
|
||||
total_tokens: 2_250,
|
||||
};
|
||||
|
||||
let captured_at = chrono::Local
|
||||
.with_ymd_and_hms(2024, 1, 2, 3, 4, 5)
|
||||
.single()
|
||||
.expect("timestamp");
|
||||
let snapshot = RateLimitSnapshot {
|
||||
primary: Some(RateLimitWindow {
|
||||
used_percent: 72.5,
|
||||
window_minutes: Some(300),
|
||||
resets_at: Some(reset_at_from(&captured_at, 600)),
|
||||
}),
|
||||
secondary: Some(RateLimitWindow {
|
||||
used_percent: 40.0,
|
||||
window_minutes: Some(10_080),
|
||||
resets_at: Some(reset_at_from(&captured_at, 1_800)),
|
||||
}),
|
||||
};
|
||||
let rate_display = rate_limit_snapshot_display(&snapshot, captured_at);
|
||||
let now = captured_at + ChronoDuration::minutes(20);
|
||||
|
||||
let composite = new_status_output(
|
||||
&config,
|
||||
&usage,
|
||||
Some(&usage),
|
||||
&None,
|
||||
Some(&rate_display),
|
||||
now,
|
||||
);
|
||||
let composite = new_status_output(&config, &usage, Some(&usage), &None, Some(&rate_display));
|
||||
let mut rendered_lines = render_lines(&composite.display_lines(80));
|
||||
if cfg!(windows) {
|
||||
for line in &mut rendered_lines {
|
||||
@@ -404,12 +314,7 @@ fn status_context_window_uses_last_usage() {
|
||||
total_tokens: 13_679,
|
||||
};
|
||||
|
||||
let now = chrono::Local
|
||||
.with_ymd_and_hms(2024, 6, 1, 12, 0, 0)
|
||||
.single()
|
||||
.expect("timestamp");
|
||||
|
||||
let composite = new_status_output(&config, &total_usage, Some(&last_usage), &None, None, now);
|
||||
let composite = new_status_output(&config, &total_usage, Some(&last_usage), &None, None);
|
||||
let rendered_lines = render_lines(&composite.display_lines(80));
|
||||
let context_line = rendered_lines
|
||||
.into_iter()
|
||||
|
||||
@@ -107,6 +107,12 @@ impl Tokenizer {
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Debug for Tokenizer {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.debug_struct("Tokenizer").finish()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
11
docs/faq.md
11
docs/faq.md
@@ -42,14 +42,3 @@ Running Codex directly on Windows may work, but is not officially supported. We
|
||||
### Where should I start after installation?
|
||||
|
||||
Follow the quick setup in [Install & build](./install.md) and then jump into [Getting started](./getting-started.md) for interactive usage tips, prompt examples, and AGENTS.md guidance.
|
||||
|
||||
### `brew upgrade codex` isn't upgrading me
|
||||
|
||||
If you're running Codex v0.46.0 or older, `brew upgrade codex` will not move you to the latest version because we migrated from a Homebrew formula to a cask. To upgrade, uninstall the existing oudated formula and then install the new cask:
|
||||
|
||||
```bash
|
||||
brew uninstall --formula codex
|
||||
brew install --cask codex
|
||||
```
|
||||
|
||||
After reinstalling, `brew upgrade --cask codex` will keep future releases up to date.
|
||||
|
||||
Reference in New Issue
Block a user