Compare commits

...

3 Commits

Author SHA1 Message Date
rhan-oai
28611de7b4 core: emit responses api call analytics 2026-04-21 18:49:28 -07:00
rhan-oai
460c00c603 analytics: reduce responses api call facts 2026-04-21 18:48:38 -07:00
rhan-oai
240de45639 analytics: add responses api call schema 2026-04-21 18:48:38 -07:00
8 changed files with 1051 additions and 1 deletions

View File

@@ -7,6 +7,8 @@ use crate::events::CodexCompactionEventRequest;
use crate::events::CodexHookRunEventRequest;
use crate::events::CodexPluginEventRequest;
use crate::events::CodexPluginUsedEventRequest;
use crate::events::CodexResponsesApiCallEventParams;
use crate::events::CodexResponsesApiCallEventRequest;
use crate::events::CodexRuntimeMetadata;
use crate::events::CodexTurnEventRequest;
use crate::events::GuardianApprovalRequestSource;
@@ -29,6 +31,11 @@ use crate::facts::AppInvocation;
use crate::facts::AppMentionedInput;
use crate::facts::AppUsedInput;
use crate::facts::CodexCompactionEvent;
use crate::facts::CodexResponseItemType;
use crate::facts::CodexResponsesApiCallFact;
use crate::facts::CodexResponsesApiCallStatus;
use crate::facts::CodexResponsesApiItemMetadata;
use crate::facts::CodexResponsesApiItemPhase;
use crate::facts::CompactionImplementation;
use crate::facts::CompactionPhase;
use crate::facts::CompactionReason;
@@ -91,6 +98,7 @@ use codex_plugin::PluginTelemetryMetadata;
use codex_protocol::approvals::NetworkApprovalProtocol;
use codex_protocol::config_types::ApprovalsReviewer;
use codex_protocol::config_types::ModeKind;
use codex_protocol::models::MessagePhase;
use codex_protocol::protocol::AskForApproval;
use codex_protocol::protocol::HookEventName;
use codex_protocol::protocol::HookRunStatus;
@@ -273,6 +281,25 @@ fn sample_turn_token_usage_fact(thread_id: &str, turn_id: &str) -> TurnTokenUsag
}
}
fn sample_responses_api_item_metadata(
item_phase: CodexResponsesApiItemPhase,
item_index: usize,
) -> CodexResponsesApiItemMetadata {
CodexResponsesApiItemMetadata {
item_phase,
item_index,
response_item_type: CodexResponseItemType::Message,
role: Some("assistant".to_string()),
status: Some("completed".to_string()),
message_phase: Some(MessagePhase::FinalAnswer),
call_id: None,
tool_name: None,
payload_bytes: Some(12),
text_part_count: Some(1),
image_part_count: Some(0),
}
}
fn sample_turn_completed_notification(
thread_id: &str,
turn_id: &str,
@@ -752,6 +779,102 @@ fn compaction_event_serializes_expected_shape() {
);
}
#[test]
fn responses_api_call_event_serializes_expected_shape() {
let event = TrackEventRequest::ResponsesApiCall(Box::new(CodexResponsesApiCallEventRequest {
event_type: "codex_responses_api_call_event",
event_params: CodexResponsesApiCallEventParams {
thread_id: "thread-1".to_string(),
turn_id: "turn-1".to_string(),
ephemeral: false,
thread_source: Some("user".to_string()),
initialization_mode: ThreadInitializationMode::New,
subagent_source: None,
parent_thread_id: None,
app_server_client: sample_app_server_client_metadata(),
runtime: sample_runtime_metadata(),
responses_id: Some("resp_123".to_string()),
turn_responses_call_index: 2,
model: Some("gpt-5".to_string()),
model_provider: Some("openai".to_string()),
reasoning_effort: Some("high".to_string()),
status: CodexResponsesApiCallStatus::Completed,
error: None,
started_at: 100,
completed_at: Some(102),
duration_ms: Some(2345),
input_item_count: 1,
output_item_count: 1,
input_tokens: Some(10),
cached_input_tokens: Some(4),
output_tokens: Some(20),
reasoning_output_tokens: Some(3),
total_tokens: Some(30),
items: vec![
sample_responses_api_item_metadata(
CodexResponsesApiItemPhase::Input,
/*item_index*/ 0,
),
sample_responses_api_item_metadata(
CodexResponsesApiItemPhase::Output,
/*item_index*/ 0,
),
],
},
}));
let payload = serde_json::to_value(&event).expect("serialize responses api call event");
assert_eq!(payload["event_type"], "codex_responses_api_call_event");
assert_eq!(payload["event_params"]["thread_id"], "thread-1");
assert_eq!(payload["event_params"]["turn_id"], "turn-1");
assert_eq!(payload["event_params"]["ephemeral"], false);
assert_eq!(payload["event_params"]["thread_source"], "user");
assert_eq!(payload["event_params"]["initialization_mode"], "new");
assert_eq!(
payload["event_params"]["app_server_client"]["product_client_id"],
DEFAULT_ORIGINATOR
);
assert_eq!(
payload["event_params"]["app_server_client"]["rpc_transport"],
"stdio"
);
assert_eq!(
payload["event_params"]["runtime"]["codex_rs_version"],
"0.1.0"
);
assert_eq!(payload["event_params"]["responses_id"], "resp_123");
assert_eq!(payload["event_params"]["turn_responses_call_index"], 2);
assert_eq!(payload["event_params"]["model"], "gpt-5");
assert_eq!(payload["event_params"]["model_provider"], "openai");
assert_eq!(payload["event_params"]["reasoning_effort"], "high");
assert_eq!(payload["event_params"]["status"], "completed");
assert_eq!(payload["event_params"]["error"], json!(null));
assert_eq!(payload["event_params"]["started_at"], 100);
assert_eq!(payload["event_params"]["completed_at"], 102);
assert_eq!(payload["event_params"]["duration_ms"], 2345);
assert_eq!(payload["event_params"]["input_item_count"], 1);
assert_eq!(payload["event_params"]["output_item_count"], 1);
assert_eq!(payload["event_params"]["input_tokens"], 10);
assert_eq!(payload["event_params"]["cached_input_tokens"], 4);
assert_eq!(payload["event_params"]["output_tokens"], 20);
assert_eq!(payload["event_params"]["reasoning_output_tokens"], 3);
assert_eq!(payload["event_params"]["total_tokens"], 30);
assert_eq!(payload["event_params"]["items"][0]["item_phase"], "input");
assert_eq!(
payload["event_params"]["items"][0]["response_item_type"],
"message"
);
assert_eq!(payload["event_params"]["items"][0]["role"], "assistant");
assert_eq!(
payload["event_params"]["items"][0]["message_phase"],
"final_answer"
);
assert_eq!(payload["event_params"]["items"][0]["text_part_count"], 1);
assert_eq!(payload["event_params"]["items"][0]["image_part_count"], 0);
assert_eq!(payload["event_params"]["items"][1]["item_phase"], "output");
}
#[test]
fn app_used_dedupe_is_keyed_by_turn_and_connector() {
let (sender, _receiver) = mpsc::channel(1);
@@ -1186,6 +1309,88 @@ async fn guardian_review_event_ingests_custom_fact_with_optional_target_item() {
assert_eq!(payload[0]["event_params"]["review_timeout_ms"], 90_000);
}
#[tokio::test]
async fn responses_api_call_event_ingests_custom_fact() {
let mut reducer = AnalyticsReducer::default();
let mut events = Vec::new();
ingest_turn_prerequisites(
&mut reducer,
&mut events,
/*include_initialize*/ true,
/*include_resolved_config*/ true,
/*include_started*/ false,
/*include_token_usage*/ false,
)
.await;
events.clear();
reducer
.ingest(
AnalyticsFact::Custom(CustomAnalyticsFact::ResponsesApiCall(Box::new(
CodexResponsesApiCallFact {
thread_id: "thread-2".to_string(),
turn_id: "turn-2".to_string(),
responses_id: Some("resp_456".to_string()),
turn_responses_call_index: 1,
status: CodexResponsesApiCallStatus::Completed,
error: None,
started_at: 100,
completed_at: Some(101),
duration_ms: Some(1200),
input_item_count: 1,
output_item_count: 1,
input_tokens: Some(123),
cached_input_tokens: Some(45),
output_tokens: Some(140),
reasoning_output_tokens: Some(13),
total_tokens: Some(321),
items: vec![
sample_responses_api_item_metadata(
CodexResponsesApiItemPhase::Input,
/*item_index*/ 0,
),
sample_responses_api_item_metadata(
CodexResponsesApiItemPhase::Output,
/*item_index*/ 0,
),
],
},
))),
&mut events,
)
.await;
let payload = serde_json::to_value(&events).expect("serialize events");
assert_eq!(payload.as_array().expect("events array").len(), 1);
assert_eq!(payload[0]["event_type"], "codex_responses_api_call_event");
assert_eq!(payload[0]["event_params"]["thread_id"], "thread-2");
assert_eq!(payload[0]["event_params"]["turn_id"], "turn-2");
assert_eq!(payload[0]["event_params"]["ephemeral"], false);
assert_eq!(payload[0]["event_params"]["thread_source"], "user");
assert_eq!(payload[0]["event_params"]["initialization_mode"], "new");
assert_eq!(
payload[0]["event_params"]["app_server_client"]["product_client_id"],
"codex-tui"
);
assert_eq!(payload[0]["event_params"]["runtime"]["runtime_os"], "macos");
assert_eq!(payload[0]["event_params"]["responses_id"], "resp_456");
assert_eq!(payload[0]["event_params"]["turn_responses_call_index"], 1);
assert_eq!(payload[0]["event_params"]["model"], "gpt-5");
assert_eq!(payload[0]["event_params"]["model_provider"], "openai");
assert_eq!(payload[0]["event_params"]["reasoning_effort"], json!(null));
assert_eq!(payload[0]["event_params"]["status"], "completed");
assert_eq!(payload[0]["event_params"]["input_tokens"], 123);
assert_eq!(
payload[0]["event_params"]["items"][0]["item_phase"],
"input"
);
assert_eq!(
payload[0]["event_params"]["items"][1]["item_phase"],
"output"
);
}
#[test]
fn subagent_thread_started_review_serializes_expected_shape() {
let event = TrackEventRequest::ThreadInitialized(subagent_thread_started_event_request(

View File

@@ -8,6 +8,8 @@ use crate::facts::AnalyticsJsonRpcError;
use crate::facts::AppInvocation;
use crate::facts::AppMentionedInput;
use crate::facts::AppUsedInput;
use crate::facts::CodexResponsesApiCallFact;
use crate::facts::CodexResponsesApiCallInput;
use crate::facts::CustomAnalyticsFact;
use crate::facts::HookRunFact;
use crate::facts::HookRunInput;
@@ -38,6 +40,7 @@ use tokio::sync::mpsc;
const ANALYTICS_EVENTS_QUEUE_SIZE: usize = 256;
const ANALYTICS_EVENTS_TIMEOUT: Duration = Duration::from_secs(10);
const ANALYTICS_EVENT_DEDUPE_MAX_KEYS: usize = 4096;
const RESPONSES_API_ERROR_MAX_BYTES: usize = 1024;
#[derive(Clone)]
pub(crate) struct AnalyticsEventsQueue {
@@ -214,6 +217,40 @@ impl AnalyticsEventsClient {
)));
}
pub fn track_responses_api_call(
&self,
tracking: TrackEventsContext,
input: CodexResponsesApiCallInput,
) {
let token_usage = input.token_usage;
let error = input.error.map(truncate_responses_api_error);
let event = CodexResponsesApiCallFact {
thread_id: tracking.thread_id,
turn_id: tracking.turn_id,
responses_id: input.responses_id,
turn_responses_call_index: input.turn_responses_call_index,
status: input.status,
error,
started_at: input.started_at,
completed_at: input.completed_at,
duration_ms: input.duration_ms,
input_item_count: input.input_item_count,
output_item_count: input.output_item_count,
input_tokens: token_usage.as_ref().map(|usage| usage.input_tokens),
cached_input_tokens: token_usage.as_ref().map(|usage| usage.cached_input_tokens),
output_tokens: token_usage.as_ref().map(|usage| usage.output_tokens),
reasoning_output_tokens: token_usage
.as_ref()
.map(|usage| usage.reasoning_output_tokens),
total_tokens: token_usage.as_ref().map(|usage| usage.total_tokens),
items: input.items,
};
self.record_fact(AnalyticsFact::Custom(
CustomAnalyticsFact::ResponsesApiCall(Box::new(event)),
));
}
pub fn track_turn_resolved_config(&self, fact: TurnResolvedConfigFact) {
self.record_fact(AnalyticsFact::Custom(
CustomAnalyticsFact::TurnResolvedConfig(Box::new(fact)),
@@ -296,6 +333,18 @@ impl AnalyticsEventsClient {
}
}
fn truncate_responses_api_error(mut error: String) -> String {
if error.len() <= RESPONSES_API_ERROR_MAX_BYTES {
return error;
}
let mut truncate_at = RESPONSES_API_ERROR_MAX_BYTES;
while !error.is_char_boundary(truncate_at) {
truncate_at -= 1;
}
error.truncate(truncate_at);
error
}
async fn send_track_events(
auth_manager: &Arc<AuthManager>,
base_url: &str,

View File

@@ -1,5 +1,7 @@
use crate::facts::AppInvocation;
use crate::facts::CodexCompactionEvent;
use crate::facts::CodexResponsesApiCallStatus;
use crate::facts::CodexResponsesApiItemMetadata;
use crate::facts::CompactionImplementation;
use crate::facts::CompactionPhase;
use crate::facts::CompactionReason;
@@ -55,6 +57,7 @@ pub(crate) enum TrackEventRequest {
AppUsed(CodexAppUsedEventRequest),
HookRun(CodexHookRunEventRequest),
Compaction(Box<CodexCompactionEventRequest>),
ResponsesApiCall(Box<CodexResponsesApiCallEventRequest>),
TurnEvent(Box<CodexTurnEventRequest>),
TurnSteer(CodexTurnSteerEventRequest),
PluginUsed(CodexPluginUsedEventRequest),
@@ -311,6 +314,43 @@ pub(crate) struct CodexCompactionEventRequest {
pub(crate) event_params: CodexCompactionEventParams,
}
#[derive(Serialize)]
pub(crate) struct CodexResponsesApiCallEventParams {
pub(crate) thread_id: String,
pub(crate) turn_id: String,
pub(crate) ephemeral: bool,
pub(crate) thread_source: Option<String>,
pub(crate) initialization_mode: ThreadInitializationMode,
pub(crate) subagent_source: Option<String>,
pub(crate) parent_thread_id: Option<String>,
pub(crate) app_server_client: CodexAppServerClientMetadata,
pub(crate) runtime: CodexRuntimeMetadata,
pub(crate) responses_id: Option<String>,
pub(crate) turn_responses_call_index: u64,
pub(crate) model: Option<String>,
pub(crate) model_provider: Option<String>,
pub(crate) reasoning_effort: Option<String>,
pub(crate) status: CodexResponsesApiCallStatus,
pub(crate) error: Option<String>,
pub(crate) started_at: u64,
pub(crate) completed_at: Option<u64>,
pub(crate) duration_ms: Option<u64>,
pub(crate) input_item_count: usize,
pub(crate) output_item_count: usize,
pub(crate) input_tokens: Option<i64>,
pub(crate) cached_input_tokens: Option<i64>,
pub(crate) output_tokens: Option<i64>,
pub(crate) reasoning_output_tokens: Option<i64>,
pub(crate) total_tokens: Option<i64>,
pub(crate) items: Vec<CodexResponsesApiItemMetadata>,
}
#[derive(Serialize)]
pub(crate) struct CodexResponsesApiCallEventRequest {
pub(crate) event_type: &'static str,
pub(crate) event_params: CodexResponsesApiCallEventParams,
}
#[derive(Serialize)]
pub(crate) struct CodexTurnEventParams {
pub(crate) thread_id: String,

View File

@@ -13,6 +13,7 @@ use codex_protocol::config_types::ModeKind;
use codex_protocol::config_types::Personality;
use codex_protocol::config_types::ReasoningSummary;
use codex_protocol::config_types::ServiceTier;
use codex_protocol::models::MessagePhase;
use codex_protocol::openai_models::ReasoningEffort;
use codex_protocol::protocol::AskForApproval;
use codex_protocol::protocol::HookEventName;
@@ -262,6 +263,91 @@ pub struct CodexCompactionEvent {
pub duration_ms: Option<u64>,
}
#[derive(Clone, Copy, Debug, Serialize, PartialEq, Eq)]
#[serde(rename_all = "snake_case")]
pub enum CodexResponsesApiCallStatus {
Completed,
Failed,
Interrupted,
}
#[derive(Clone, Copy, Debug, Serialize, PartialEq, Eq)]
#[serde(rename_all = "lowercase")]
pub enum CodexResponsesApiItemPhase {
Input,
Output,
}
#[derive(Clone, Copy, Debug, Serialize, PartialEq, Eq)]
#[serde(rename_all = "snake_case")]
pub enum CodexResponseItemType {
Message,
Reasoning,
LocalShellCall,
FunctionCall,
FunctionCallOutput,
CustomToolCall,
CustomToolCallOutput,
ToolSearchCall,
ToolSearchOutput,
WebSearchCall,
ImageGenerationCall,
GhostSnapshot,
Compaction,
Other,
}
#[derive(Clone, Debug, Serialize, PartialEq, Eq)]
pub struct CodexResponsesApiItemMetadata {
pub item_phase: CodexResponsesApiItemPhase,
pub item_index: usize,
pub response_item_type: CodexResponseItemType,
pub role: Option<String>,
pub status: Option<String>,
pub message_phase: Option<MessagePhase>,
pub call_id: Option<String>,
pub tool_name: Option<String>,
pub payload_bytes: Option<i64>,
pub text_part_count: Option<usize>,
pub image_part_count: Option<usize>,
}
#[derive(Clone, Debug)]
pub struct CodexResponsesApiCallInput {
pub responses_id: Option<String>,
pub turn_responses_call_index: u64,
pub status: CodexResponsesApiCallStatus,
pub error: Option<String>,
pub started_at: u64,
pub completed_at: Option<u64>,
pub duration_ms: Option<u64>,
pub input_item_count: usize,
pub output_item_count: usize,
pub items: Vec<CodexResponsesApiItemMetadata>,
pub token_usage: Option<TokenUsage>,
}
#[derive(Clone, Debug)]
pub struct CodexResponsesApiCallFact {
pub thread_id: String,
pub turn_id: String,
pub responses_id: Option<String>,
pub turn_responses_call_index: u64,
pub status: CodexResponsesApiCallStatus,
pub error: Option<String>,
pub started_at: u64,
pub completed_at: Option<u64>,
pub duration_ms: Option<u64>,
pub input_item_count: usize,
pub output_item_count: usize,
pub input_tokens: Option<i64>,
pub cached_input_tokens: Option<i64>,
pub output_tokens: Option<i64>,
pub reasoning_output_tokens: Option<i64>,
pub total_tokens: Option<i64>,
pub items: Vec<CodexResponsesApiItemMetadata>,
}
#[allow(dead_code)]
pub(crate) enum AnalyticsFact {
Initialize {
@@ -295,6 +381,7 @@ pub(crate) enum AnalyticsFact {
pub(crate) enum CustomAnalyticsFact {
SubAgentThreadStarted(SubAgentThreadStartedInput),
Compaction(Box<CodexCompactionEvent>),
ResponsesApiCall(Box<CodexResponsesApiCallFact>),
GuardianReview(Box<GuardianReviewEventParams>),
TurnResolvedConfig(Box<TurnResolvedConfigFact>),
TurnTokenUsage(Box<TurnTokenUsageFact>),

View File

@@ -2,7 +2,10 @@ mod client;
mod events;
mod facts;
mod reducer;
mod response_items;
use codex_protocol::models::ResponseItem;
use serde::Serialize;
use std::time::SystemTime;
use std::time::UNIX_EPOCH;
@@ -18,6 +21,9 @@ pub use events::GuardianReviewedAction;
pub use facts::AnalyticsJsonRpcError;
pub use facts::AppInvocation;
pub use facts::CodexCompactionEvent;
pub use facts::CodexResponsesApiCallInput;
pub use facts::CodexResponsesApiCallStatus;
pub use facts::CodexResponsesApiItemMetadata;
pub use facts::CodexTurnSteerEvent;
pub use facts::CompactionImplementation;
pub use facts::CompactionPhase;
@@ -49,3 +55,41 @@ pub fn now_unix_seconds() -> u64 {
.unwrap_or_default()
.as_secs()
}
pub fn responses_api_input_items_metadata(
items: &[ResponseItem],
) -> Vec<CodexResponsesApiItemMetadata> {
response_items::response_items_metadata(facts::CodexResponsesApiItemPhase::Input, items)
}
pub fn responses_api_output_item_metadata(
item_index: usize,
item: &ResponseItem,
) -> CodexResponsesApiItemMetadata {
response_items::response_item_metadata(
facts::CodexResponsesApiItemPhase::Output,
item_index,
item,
)
}
pub(crate) fn serialized_string<T: Serialize>(value: &T) -> Option<String> {
match serde_json::to_value(value).ok()? {
serde_json::Value::String(value) => Some(value),
value => Some(value.to_string()),
}
}
pub(crate) fn serialized_bytes<T: Serialize>(value: &T) -> Option<i64> {
serde_json::to_string(value)
.ok()
.map(|value| byte_len(&value))
}
pub(crate) fn nonzero_i64(value: i64) -> Option<i64> {
(value > 0).then_some(value)
}
pub(crate) fn byte_len(value: &str) -> i64 {
i64::try_from(value.len()).unwrap_or(i64::MAX)
}

View File

@@ -6,6 +6,8 @@ use crate::events::CodexCompactionEventRequest;
use crate::events::CodexHookRunEventRequest;
use crate::events::CodexPluginEventRequest;
use crate::events::CodexPluginUsedEventRequest;
use crate::events::CodexResponsesApiCallEventParams;
use crate::events::CodexResponsesApiCallEventRequest;
use crate::events::CodexRuntimeMetadata;
use crate::events::CodexTurnEventParams;
use crate::events::CodexTurnEventRequest;
@@ -33,6 +35,7 @@ use crate::facts::AnalyticsJsonRpcError;
use crate::facts::AppMentionedInput;
use crate::facts::AppUsedInput;
use crate::facts::CodexCompactionEvent;
use crate::facts::CodexResponsesApiCallFact;
use crate::facts::CustomAnalyticsFact;
use crate::facts::HookRunInput;
use crate::facts::PluginState;
@@ -202,6 +205,9 @@ impl AnalyticsReducer {
CustomAnalyticsFact::Compaction(input) => {
self.ingest_compaction(*input, out);
}
CustomAnalyticsFact::ResponsesApiCall(input) => {
self.ingest_responses_api_call(*input, out);
}
CustomAnalyticsFact::GuardianReview(input) => {
self.ingest_guardian_review(*input, out);
}
@@ -739,6 +745,91 @@ impl AnalyticsReducer {
)));
}
fn ingest_responses_api_call(
&mut self,
input: CodexResponsesApiCallFact,
out: &mut Vec<TrackEventRequest>,
) {
let Some(connection_id) = self.thread_connections.get(&input.thread_id) else {
tracing::warn!(
thread_id = %input.thread_id,
turn_id = %input.turn_id,
"dropping responses api call analytics event: missing thread connection metadata"
);
return;
};
let Some(connection_state) = self.connections.get(connection_id) else {
tracing::warn!(
thread_id = %input.thread_id,
turn_id = %input.turn_id,
connection_id,
"dropping responses api call analytics event: missing connection metadata"
);
return;
};
let Some(thread_metadata) = self.thread_metadata.get(&input.thread_id) else {
tracing::warn!(
thread_id = %input.thread_id,
turn_id = %input.turn_id,
"dropping responses api call analytics event: missing thread lifecycle metadata"
);
return;
};
let Some(turn_state) = self.turns.get(&input.turn_id) else {
tracing::warn!(
thread_id = %input.thread_id,
turn_id = %input.turn_id,
"dropping responses api call analytics event: missing turn metadata"
);
return;
};
let Some(resolved_config) = turn_state.resolved_config.as_ref() else {
tracing::warn!(
thread_id = %input.thread_id,
turn_id = %input.turn_id,
"dropping responses api call analytics event: missing turn resolved config"
);
return;
};
out.push(TrackEventRequest::ResponsesApiCall(Box::new(
CodexResponsesApiCallEventRequest {
event_type: "codex_responses_api_call_event",
event_params: CodexResponsesApiCallEventParams {
thread_id: input.thread_id,
turn_id: input.turn_id,
ephemeral: resolved_config.ephemeral,
thread_source: thread_metadata.thread_source.map(str::to_string),
initialization_mode: thread_metadata.initialization_mode,
subagent_source: thread_metadata.subagent_source.clone(),
parent_thread_id: thread_metadata.parent_thread_id.clone(),
app_server_client: connection_state.app_server_client.clone(),
runtime: connection_state.runtime.clone(),
responses_id: input.responses_id,
turn_responses_call_index: input.turn_responses_call_index,
model: Some(resolved_config.model.clone()),
model_provider: Some(resolved_config.model_provider.clone()),
reasoning_effort: resolved_config
.reasoning_effort
.map(|value| value.to_string()),
status: input.status,
error: input.error,
started_at: input.started_at,
completed_at: input.completed_at,
duration_ms: input.duration_ms,
input_item_count: input.input_item_count,
output_item_count: input.output_item_count,
input_tokens: input.input_tokens,
cached_input_tokens: input.cached_input_tokens,
output_tokens: input.output_tokens,
reasoning_output_tokens: input.reasoning_output_tokens,
total_tokens: input.total_tokens,
items: input.items,
},
},
)));
}
fn ingest_turn_steer_response(
&mut self,
connection_id: u64,

View File

@@ -0,0 +1,366 @@
use crate::byte_len;
use crate::facts::CodexResponseItemType;
use crate::facts::CodexResponsesApiItemMetadata;
use crate::facts::CodexResponsesApiItemPhase;
use crate::nonzero_i64;
use crate::serialized_bytes;
use crate::serialized_string;
use codex_protocol::models::ContentItem;
use codex_protocol::models::FunctionCallOutputBody;
use codex_protocol::models::FunctionCallOutputContentItem;
use codex_protocol::models::FunctionCallOutputPayload;
use codex_protocol::models::ReasoningItemContent;
use codex_protocol::models::ReasoningItemReasoningSummary;
use codex_protocol::models::ResponseItem;
pub(crate) fn response_items_metadata(
phase: CodexResponsesApiItemPhase,
items: &[ResponseItem],
) -> Vec<CodexResponsesApiItemMetadata> {
items
.iter()
.enumerate()
.map(|(item_index, item)| response_item_metadata(phase, item_index, item))
.collect()
}
pub(crate) fn response_item_metadata(
item_phase: CodexResponsesApiItemPhase,
item_index: usize,
item: &ResponseItem,
) -> CodexResponsesApiItemMetadata {
let mut metadata = new_metadata(item_phase, item_index, response_item_type(item));
match item {
ResponseItem::Message {
role,
content,
phase,
..
} => {
metadata.role = Some(role.clone());
metadata.message_phase = phase.clone();
metadata.payload_bytes = nonzero_i64(message_content_text_bytes(content));
let (text_part_count, image_part_count) = message_content_part_counts(content);
metadata.text_part_count = Some(text_part_count);
metadata.image_part_count = Some(image_part_count);
}
ResponseItem::Reasoning {
summary,
content,
encrypted_content,
..
} => {
metadata.payload_bytes = encrypted_content
.as_ref()
.map(|value| byte_len(value))
.or_else(|| nonzero_i64(reasoning_content_bytes(summary, content)));
metadata.text_part_count =
Some(summary.len() + content.as_ref().map(std::vec::Vec::len).unwrap_or_default());
metadata.image_part_count = Some(0);
}
ResponseItem::LocalShellCall {
call_id,
status,
action,
..
} => {
metadata.call_id = call_id.clone();
metadata.tool_name = Some("local_shell".to_string());
metadata.status = serialized_string(status);
metadata.payload_bytes = serialized_bytes(action);
}
ResponseItem::FunctionCall {
name,
arguments,
call_id,
..
} => {
metadata.call_id = Some(call_id.clone());
metadata.tool_name = Some(name.clone());
metadata.payload_bytes = Some(byte_len(arguments));
}
ResponseItem::ToolSearchCall {
call_id,
status,
arguments,
..
} => {
metadata.call_id = call_id.clone();
metadata.tool_name = Some("tool_search".to_string());
metadata.status = status.clone();
metadata.payload_bytes = serialized_bytes(arguments);
}
ResponseItem::FunctionCallOutput { call_id, output } => {
metadata.call_id = Some(call_id.clone());
metadata.payload_bytes = function_call_output_bytes(output);
let (text_part_count, image_part_count) = function_call_output_part_counts(output);
metadata.text_part_count = text_part_count;
metadata.image_part_count = image_part_count;
}
ResponseItem::CustomToolCall {
status,
call_id,
name,
input,
..
} => {
metadata.call_id = Some(call_id.clone());
metadata.tool_name = Some(name.clone());
metadata.status = status.clone();
metadata.payload_bytes = Some(byte_len(input));
}
ResponseItem::CustomToolCallOutput {
call_id,
name,
output,
} => {
metadata.call_id = Some(call_id.clone());
metadata.tool_name = name.clone();
metadata.payload_bytes = function_call_output_bytes(output);
let (text_part_count, image_part_count) = function_call_output_part_counts(output);
metadata.text_part_count = text_part_count;
metadata.image_part_count = image_part_count;
}
ResponseItem::ToolSearchOutput {
call_id,
status,
tools,
..
} => {
metadata.call_id = call_id.clone();
metadata.tool_name = Some("tool_search".to_string());
metadata.status = Some(status.clone());
metadata.payload_bytes = serialized_bytes(tools);
}
ResponseItem::WebSearchCall { status, action, .. } => {
metadata.tool_name = Some("web_search".to_string());
metadata.status = status.clone();
metadata.payload_bytes = action.as_ref().and_then(serialized_bytes);
}
ResponseItem::ImageGenerationCall {
id,
status,
revised_prompt,
result,
} => {
metadata.call_id = Some(id.clone());
metadata.tool_name = Some("image_generation".to_string());
metadata.status = Some(status.clone());
metadata.payload_bytes = nonzero_i64(byte_len(result))
.or_else(|| revised_prompt.as_ref().map(|value| byte_len(value)));
}
ResponseItem::Compaction { encrypted_content } => {
metadata.payload_bytes = Some(byte_len(encrypted_content));
}
ResponseItem::GhostSnapshot { .. } | ResponseItem::Other => {}
}
metadata
}
fn new_metadata(
item_phase: CodexResponsesApiItemPhase,
item_index: usize,
response_item_type: CodexResponseItemType,
) -> CodexResponsesApiItemMetadata {
CodexResponsesApiItemMetadata {
item_phase,
item_index,
response_item_type,
role: None,
status: None,
message_phase: None,
call_id: None,
tool_name: None,
payload_bytes: None,
text_part_count: None,
image_part_count: None,
}
}
fn response_item_type(item: &ResponseItem) -> CodexResponseItemType {
match item {
ResponseItem::Message { .. } => CodexResponseItemType::Message,
ResponseItem::Reasoning { .. } => CodexResponseItemType::Reasoning,
ResponseItem::LocalShellCall { .. } => CodexResponseItemType::LocalShellCall,
ResponseItem::FunctionCall { .. } => CodexResponseItemType::FunctionCall,
ResponseItem::ToolSearchCall { .. } => CodexResponseItemType::ToolSearchCall,
ResponseItem::FunctionCallOutput { .. } => CodexResponseItemType::FunctionCallOutput,
ResponseItem::CustomToolCall { .. } => CodexResponseItemType::CustomToolCall,
ResponseItem::CustomToolCallOutput { .. } => CodexResponseItemType::CustomToolCallOutput,
ResponseItem::ToolSearchOutput { .. } => CodexResponseItemType::ToolSearchOutput,
ResponseItem::WebSearchCall { .. } => CodexResponseItemType::WebSearchCall,
ResponseItem::ImageGenerationCall { .. } => CodexResponseItemType::ImageGenerationCall,
ResponseItem::GhostSnapshot { .. } => CodexResponseItemType::GhostSnapshot,
ResponseItem::Compaction { .. } => CodexResponseItemType::Compaction,
ResponseItem::Other => CodexResponseItemType::Other,
}
}
fn message_content_text_bytes(content: &[ContentItem]) -> i64 {
content
.iter()
.map(|item| match item {
ContentItem::InputText { text } | ContentItem::OutputText { text } => byte_len(text),
ContentItem::InputImage { .. } => 0,
})
.sum()
}
fn message_content_part_counts(content: &[ContentItem]) -> (usize, usize) {
let mut text_part_count = 0;
let mut image_part_count = 0;
for item in content {
match item {
ContentItem::InputText { .. } | ContentItem::OutputText { .. } => {
text_part_count += 1;
}
ContentItem::InputImage { .. } => {
image_part_count += 1;
}
}
}
(text_part_count, image_part_count)
}
fn reasoning_content_bytes(
summary: &[ReasoningItemReasoningSummary],
content: &Option<Vec<ReasoningItemContent>>,
) -> i64 {
let summary_bytes = summary
.iter()
.map(|summary| match summary {
ReasoningItemReasoningSummary::SummaryText { text } => byte_len(text),
})
.sum::<i64>();
let content_bytes = content
.as_ref()
.map(|content| {
content
.iter()
.map(|content| match content {
ReasoningItemContent::ReasoningText { text }
| ReasoningItemContent::Text { text } => byte_len(text),
})
.sum::<i64>()
})
.unwrap_or_default();
summary_bytes + content_bytes
}
fn function_call_output_bytes(output: &FunctionCallOutputPayload) -> Option<i64> {
match &output.body {
FunctionCallOutputBody::Text(text) => Some(byte_len(text)),
FunctionCallOutputBody::ContentItems(items) => serialized_bytes(items),
}
}
fn function_call_output_part_counts(
output: &FunctionCallOutputPayload,
) -> (Option<usize>, Option<usize>) {
match &output.body {
FunctionCallOutputBody::Text(_) => (Some(1), Some(0)),
FunctionCallOutputBody::ContentItems(content_items) => {
let mut text_part_count = 0;
let mut image_part_count = 0;
for item in content_items {
match item {
FunctionCallOutputContentItem::InputText { .. } => {
text_part_count += 1;
}
FunctionCallOutputContentItem::InputImage { .. } => {
image_part_count += 1;
}
}
}
(Some(text_part_count), Some(image_part_count))
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use codex_protocol::models::ImageDetail;
use codex_protocol::models::MessagePhase;
#[test]
fn maps_message_metadata() {
let items = vec![ResponseItem::Message {
id: None,
role: "assistant".to_string(),
content: vec![
ContentItem::OutputText {
text: "hello".to_string(),
},
ContentItem::InputImage {
image_url: "data:image/png;base64,abc".to_string(),
detail: None,
},
],
end_turn: None,
phase: Some(MessagePhase::FinalAnswer),
}];
let metadata = response_items_metadata(CodexResponsesApiItemPhase::Output, &items);
assert_eq!(metadata[0].item_phase, CodexResponsesApiItemPhase::Output);
assert_eq!(
metadata[0].response_item_type,
CodexResponseItemType::Message
);
assert_eq!(metadata[0].role.as_deref(), Some("assistant"));
assert_eq!(metadata[0].message_phase, Some(MessagePhase::FinalAnswer));
assert_eq!(metadata[0].payload_bytes, Some(5));
assert_eq!(metadata[0].text_part_count, Some(1));
assert_eq!(metadata[0].image_part_count, Some(1));
}
#[test]
fn maps_tool_call_output_metadata() {
let items = vec![ResponseItem::CustomToolCallOutput {
call_id: "call_1".to_string(),
name: Some("custom_tool".to_string()),
output: FunctionCallOutputPayload::from_content_items(vec![
FunctionCallOutputContentItem::InputText {
text: "result".to_string(),
},
FunctionCallOutputContentItem::InputImage {
image_url: "https://example.test/image.png".to_string(),
detail: Some(ImageDetail::High),
},
]),
}];
let metadata = response_items_metadata(CodexResponsesApiItemPhase::Output, &items);
assert_eq!(
metadata[0].response_item_type,
CodexResponseItemType::CustomToolCallOutput
);
assert_eq!(metadata[0].call_id.as_deref(), Some("call_1"));
assert_eq!(metadata[0].tool_name.as_deref(), Some("custom_tool"));
assert!(metadata[0].payload_bytes.unwrap_or_default() > 0);
assert_eq!(metadata[0].text_part_count, Some(1));
assert_eq!(metadata[0].image_part_count, Some(1));
}
#[test]
fn maps_text_tool_call_output_metadata() {
let items = vec![ResponseItem::FunctionCallOutput {
call_id: "call_1".to_string(),
output: FunctionCallOutputPayload::from_text("result".to_string()),
}];
let metadata = response_items_metadata(CodexResponsesApiItemPhase::Output, &items);
assert_eq!(
metadata[0].response_item_type,
CodexResponseItemType::FunctionCallOutput
);
assert_eq!(metadata[0].payload_bytes, Some(6));
assert_eq!(metadata[0].text_part_count, Some(1));
assert_eq!(metadata[0].image_part_count, Some(0));
}
}

View File

@@ -1,6 +1,7 @@
use std::collections::HashMap;
use std::collections::HashSet;
use std::sync::Arc;
use std::time::Instant;
use crate::SkillInjections;
use crate::SkillLoadOutcome;
@@ -59,11 +60,17 @@ use crate::unavailable_tool::collect_unavailable_called_tools;
use crate::util::backoff;
use crate::util::error_or_panic;
use codex_analytics::AppInvocation;
use codex_analytics::CodexResponsesApiCallInput;
use codex_analytics::CodexResponsesApiCallStatus;
use codex_analytics::CodexResponsesApiItemMetadata;
use codex_analytics::CompactionPhase;
use codex_analytics::CompactionReason;
use codex_analytics::InvocationType;
use codex_analytics::TurnResolvedConfigFact;
use codex_analytics::build_track_events_context;
use codex_analytics::now_unix_seconds;
use codex_analytics::responses_api_input_items_metadata;
use codex_analytics::responses_api_output_item_metadata;
use codex_async_utils::OrCancelExt;
use codex_features::Feature;
use codex_hooks::HookEvent;
@@ -91,6 +98,7 @@ use codex_protocol::protocol::EventMsg;
use codex_protocol::protocol::PlanDeltaEvent;
use codex_protocol::protocol::ReasoningContentDeltaEvent;
use codex_protocol::protocol::ReasoningRawContentDeltaEvent;
use codex_protocol::protocol::TokenUsage;
use codex_protocol::protocol::TurnDiffEvent;
use codex_protocol::protocol::WarningEvent;
use codex_protocol::user_input::UserInput;
@@ -406,6 +414,7 @@ pub(crate) async fn run_turn(
// 1. At the start of a turn, so the fresh user prompt in `input` gets sampled first.
// 2. After auto-compact, when model/tool continuation needs to resume before any steer.
let mut can_drain_pending_input = input.is_empty();
let mut next_turn_responses_call_index: u64 = 0;
loop {
if run_pending_session_start_hooks(&sess, &turn_context).await {
@@ -483,6 +492,7 @@ pub(crate) async fn run_turn(
Arc::clone(&turn_diff_tracker),
&mut client_session,
turn_metadata_header.as_deref(),
&mut next_turn_responses_call_index,
sampling_request_input,
&explicitly_enabled_connectors,
skills_outcome,
@@ -495,6 +505,7 @@ pub(crate) async fn run_turn(
let SamplingRequestResult {
needs_follow_up: model_needs_follow_up,
last_agent_message: sampling_request_last_agent_message,
..
} = sampling_request_output;
can_drain_pending_input = true;
let has_pending_input = sess.has_pending_input().await;
@@ -1032,6 +1043,81 @@ fn filter_deferred_dynamic_tool_spec(
}
}
struct ResponsesApiCallAttempt {
turn_responses_call_index: u64,
started_at: u64,
started_instant: Instant,
stream_started: bool,
input_item_count: usize,
output_item_count: usize,
items: Vec<CodexResponsesApiItemMetadata>,
responses_id: Option<String>,
token_usage: Option<TokenUsage>,
}
impl ResponsesApiCallAttempt {
fn new(turn_responses_call_index: u64, input_items: &[ResponseItem]) -> Self {
let input_item_count = input_items.len();
let items = responses_api_input_items_metadata(input_items);
Self {
turn_responses_call_index,
started_at: now_unix_seconds(),
started_instant: Instant::now(),
stream_started: false,
input_item_count,
output_item_count: 0,
items,
responses_id: None,
token_usage: None,
}
}
fn record_output_item(&mut self, item: &ResponseItem) {
let item_index = self.output_item_count;
self.items
.push(responses_api_output_item_metadata(item_index, item));
self.output_item_count += 1;
}
}
fn emit_responses_api_call_attempt(
sess: &Session,
turn_context: &TurnContext,
attempt: ResponsesApiCallAttempt,
status: CodexResponsesApiCallStatus,
error: Option<String>,
) {
if !sess.enabled(Feature::GeneralAnalytics) {
return;
}
if status == CodexResponsesApiCallStatus::Interrupted && !attempt.stream_started {
return;
}
let input = CodexResponsesApiCallInput {
responses_id: attempt.responses_id,
turn_responses_call_index: attempt.turn_responses_call_index,
status,
error,
started_at: attempt.started_at,
completed_at: Some(now_unix_seconds()),
duration_ms: Some(attempt.started_instant.elapsed().as_millis() as u64),
input_item_count: attempt.input_item_count,
output_item_count: attempt.output_item_count,
items: attempt.items,
token_usage: attempt.token_usage,
};
sess.services
.analytics_events_client
.track_responses_api_call(
build_track_events_context(
turn_context.model_info.slug.clone(),
sess.conversation_id.to_string(),
turn_context.sub_id.clone(),
),
input,
);
}
#[allow(clippy::too_many_arguments)]
#[instrument(level = "trace",
skip_all,
@@ -1047,6 +1133,7 @@ async fn run_sampling_request(
turn_diff_tracker: SharedTurnDiffTracker,
client_session: &mut ModelClientSession,
turn_metadata_header: Option<&str>,
next_turn_responses_call_index: &mut u64,
input: Vec<ResponseItem>,
explicitly_enabled_connectors: &HashSet<String>,
skills_outcome: Option<&SkillLoadOutcome>,
@@ -1097,6 +1184,10 @@ async fn run_sampling_request(
turn_context.as_ref(),
base_instructions.clone(),
);
let turn_responses_call_index = *next_turn_responses_call_index;
*next_turn_responses_call_index = (*next_turn_responses_call_index).saturating_add(1);
let mut responses_api_call_attempt =
ResponsesApiCallAttempt::new(turn_responses_call_index, &prompt.input);
let err = match try_run_sampling_request(
tool_runtime.clone(),
Arc::clone(&sess),
@@ -1106,15 +1197,40 @@ async fn run_sampling_request(
Arc::clone(&turn_diff_tracker),
server_model_warning_emitted_for_turn,
&prompt,
&mut responses_api_call_attempt,
cancellation_token.child_token(),
)
.await
{
Ok(output) => {
let status = if output.response_completed {
CodexResponsesApiCallStatus::Completed
} else {
CodexResponsesApiCallStatus::Interrupted
};
let error = if output.response_completed {
None
} else {
Some("stream preempted by pending input".to_string())
};
emit_responses_api_call_attempt(
sess.as_ref(),
turn_context.as_ref(),
responses_api_call_attempt,
status,
error,
);
return Ok(output);
}
Err(CodexErr::ContextWindowExceeded) => {
sess.set_total_tokens_full(&turn_context).await;
emit_responses_api_call_attempt(
sess.as_ref(),
turn_context.as_ref(),
responses_api_call_attempt,
CodexResponsesApiCallStatus::Failed,
Some("context window exceeded".to_string()),
);
return Err(CodexErr::ContextWindowExceeded);
}
Err(CodexErr::UsageLimitReached(e)) => {
@@ -1122,12 +1238,32 @@ async fn run_sampling_request(
if let Some(rate_limits) = rate_limits {
sess.update_rate_limits(&turn_context, *rate_limits).await;
}
emit_responses_api_call_attempt(
sess.as_ref(),
turn_context.as_ref(),
responses_api_call_attempt,
CodexResponsesApiCallStatus::Failed,
Some(e.to_string()),
);
return Err(CodexErr::UsageLimitReached(e));
}
Err(err) => err,
};
if !err.is_retryable() {
let status = if matches!(err, CodexErr::TurnAborted) {
CodexResponsesApiCallStatus::Interrupted
} else {
CodexResponsesApiCallStatus::Failed
};
let error = Some(format!("{err:#}"));
emit_responses_api_call_attempt(
sess.as_ref(),
turn_context.as_ref(),
responses_api_call_attempt,
status,
error,
);
return Err(err);
}
@@ -1139,6 +1275,14 @@ async fn run_sampling_request(
&turn_context.model_info,
)
{
let error = Some(format!("{err:#}"));
emit_responses_api_call_attempt(
sess.as_ref(),
turn_context.as_ref(),
responses_api_call_attempt,
CodexResponsesApiCallStatus::Failed,
error,
);
sess.send_event(
&turn_context,
EventMsg::Warning(WarningEvent {
@@ -1160,6 +1304,14 @@ async fn run_sampling_request(
warn!(
"stream disconnected - retrying sampling request ({retries}/{max_retries} in {delay:?})...",
);
let error = Some(format!("{err:#}"));
emit_responses_api_call_attempt(
sess.as_ref(),
turn_context.as_ref(),
responses_api_call_attempt,
CodexResponsesApiCallStatus::Failed,
error,
);
// In release builds, hide the first websocket retry notification to reduce noisy
// transient reconnect messages. In debug builds, keep full visibility for diagnosis.
@@ -1179,6 +1331,14 @@ async fn run_sampling_request(
}
tokio::time::sleep(delay).await;
} else {
let error = Some(format!("{err:#}"));
emit_responses_api_call_attempt(
sess.as_ref(),
turn_context.as_ref(),
responses_api_call_attempt,
CodexResponsesApiCallStatus::Failed,
error,
);
return Err(err);
}
}
@@ -1330,6 +1490,7 @@ pub(crate) async fn built_tools(
struct SamplingRequestResult {
needs_follow_up: bool,
last_agent_message: Option<String>,
response_completed: bool,
}
/// Ephemeral per-response state for streaming a single proposed plan.
@@ -1895,6 +2056,7 @@ async fn try_run_sampling_request(
turn_diff_tracker: SharedTurnDiffTracker,
server_model_warning_emitted_for_turn: &mut bool,
prompt: &Prompt,
responses_api_call_attempt: &mut ResponsesApiCallAttempt,
cancellation_token: CancellationToken,
) -> CodexResult<SamplingRequestResult> {
feedback_tags!(
@@ -1918,6 +2080,7 @@ async fn try_run_sampling_request(
.instrument(trace_span!("stream_request"))
.or_cancel(&cancellation_token)
.await??;
responses_api_call_attempt.stream_started = true;
let mut in_flight: FuturesOrdered<BoxFuture<'static, CodexResult<ResponseInputItem>>> =
FuturesOrdered::new();
let mut needs_follow_up = false;
@@ -1970,6 +2133,7 @@ async fn try_run_sampling_request(
match event {
ResponseEvent::Created => {}
ResponseEvent::OutputItemDone(item) => {
responses_api_call_attempt.record_output_item(&item);
if let Some((_, mut consumer)) = active_tool_argument_diff_consumer.take()
&& let Some(event) = consumer.flush_on_complete()
{
@@ -2049,6 +2213,7 @@ async fn try_run_sampling_request(
break Ok(SamplingRequestResult {
needs_follow_up: true,
last_agent_message,
response_completed: false,
});
}
}
@@ -2140,9 +2305,11 @@ async fn try_run_sampling_request(
sess.services.models_manager.refresh_if_new_etag(etag).await;
}
ResponseEvent::Completed {
response_id: _,
response_id,
token_usage,
} => {
responses_api_call_attempt.responses_id = Some(response_id);
responses_api_call_attempt.token_usage = token_usage.clone();
flush_assistant_text_segments_all(
&sess,
&turn_context,
@@ -2157,6 +2324,7 @@ async fn try_run_sampling_request(
break Ok(SamplingRequestResult {
needs_follow_up,
last_agent_message,
response_completed: true,
});
}
ResponseEvent::OutputTextDelta(delta) => {