mirror of
https://github.com/openai/codex.git
synced 2026-05-01 09:56:37 +00:00
Compare commits
3 Commits
ice-window
...
pr18030
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
28611de7b4 | ||
|
|
460c00c603 | ||
|
|
240de45639 |
@@ -7,6 +7,8 @@ use crate::events::CodexCompactionEventRequest;
|
||||
use crate::events::CodexHookRunEventRequest;
|
||||
use crate::events::CodexPluginEventRequest;
|
||||
use crate::events::CodexPluginUsedEventRequest;
|
||||
use crate::events::CodexResponsesApiCallEventParams;
|
||||
use crate::events::CodexResponsesApiCallEventRequest;
|
||||
use crate::events::CodexRuntimeMetadata;
|
||||
use crate::events::CodexTurnEventRequest;
|
||||
use crate::events::GuardianApprovalRequestSource;
|
||||
@@ -29,6 +31,11 @@ use crate::facts::AppInvocation;
|
||||
use crate::facts::AppMentionedInput;
|
||||
use crate::facts::AppUsedInput;
|
||||
use crate::facts::CodexCompactionEvent;
|
||||
use crate::facts::CodexResponseItemType;
|
||||
use crate::facts::CodexResponsesApiCallFact;
|
||||
use crate::facts::CodexResponsesApiCallStatus;
|
||||
use crate::facts::CodexResponsesApiItemMetadata;
|
||||
use crate::facts::CodexResponsesApiItemPhase;
|
||||
use crate::facts::CompactionImplementation;
|
||||
use crate::facts::CompactionPhase;
|
||||
use crate::facts::CompactionReason;
|
||||
@@ -91,6 +98,7 @@ use codex_plugin::PluginTelemetryMetadata;
|
||||
use codex_protocol::approvals::NetworkApprovalProtocol;
|
||||
use codex_protocol::config_types::ApprovalsReviewer;
|
||||
use codex_protocol::config_types::ModeKind;
|
||||
use codex_protocol::models::MessagePhase;
|
||||
use codex_protocol::protocol::AskForApproval;
|
||||
use codex_protocol::protocol::HookEventName;
|
||||
use codex_protocol::protocol::HookRunStatus;
|
||||
@@ -273,6 +281,25 @@ fn sample_turn_token_usage_fact(thread_id: &str, turn_id: &str) -> TurnTokenUsag
|
||||
}
|
||||
}
|
||||
|
||||
fn sample_responses_api_item_metadata(
|
||||
item_phase: CodexResponsesApiItemPhase,
|
||||
item_index: usize,
|
||||
) -> CodexResponsesApiItemMetadata {
|
||||
CodexResponsesApiItemMetadata {
|
||||
item_phase,
|
||||
item_index,
|
||||
response_item_type: CodexResponseItemType::Message,
|
||||
role: Some("assistant".to_string()),
|
||||
status: Some("completed".to_string()),
|
||||
message_phase: Some(MessagePhase::FinalAnswer),
|
||||
call_id: None,
|
||||
tool_name: None,
|
||||
payload_bytes: Some(12),
|
||||
text_part_count: Some(1),
|
||||
image_part_count: Some(0),
|
||||
}
|
||||
}
|
||||
|
||||
fn sample_turn_completed_notification(
|
||||
thread_id: &str,
|
||||
turn_id: &str,
|
||||
@@ -752,6 +779,102 @@ fn compaction_event_serializes_expected_shape() {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn responses_api_call_event_serializes_expected_shape() {
|
||||
let event = TrackEventRequest::ResponsesApiCall(Box::new(CodexResponsesApiCallEventRequest {
|
||||
event_type: "codex_responses_api_call_event",
|
||||
event_params: CodexResponsesApiCallEventParams {
|
||||
thread_id: "thread-1".to_string(),
|
||||
turn_id: "turn-1".to_string(),
|
||||
ephemeral: false,
|
||||
thread_source: Some("user".to_string()),
|
||||
initialization_mode: ThreadInitializationMode::New,
|
||||
subagent_source: None,
|
||||
parent_thread_id: None,
|
||||
app_server_client: sample_app_server_client_metadata(),
|
||||
runtime: sample_runtime_metadata(),
|
||||
responses_id: Some("resp_123".to_string()),
|
||||
turn_responses_call_index: 2,
|
||||
model: Some("gpt-5".to_string()),
|
||||
model_provider: Some("openai".to_string()),
|
||||
reasoning_effort: Some("high".to_string()),
|
||||
status: CodexResponsesApiCallStatus::Completed,
|
||||
error: None,
|
||||
started_at: 100,
|
||||
completed_at: Some(102),
|
||||
duration_ms: Some(2345),
|
||||
input_item_count: 1,
|
||||
output_item_count: 1,
|
||||
input_tokens: Some(10),
|
||||
cached_input_tokens: Some(4),
|
||||
output_tokens: Some(20),
|
||||
reasoning_output_tokens: Some(3),
|
||||
total_tokens: Some(30),
|
||||
items: vec![
|
||||
sample_responses_api_item_metadata(
|
||||
CodexResponsesApiItemPhase::Input,
|
||||
/*item_index*/ 0,
|
||||
),
|
||||
sample_responses_api_item_metadata(
|
||||
CodexResponsesApiItemPhase::Output,
|
||||
/*item_index*/ 0,
|
||||
),
|
||||
],
|
||||
},
|
||||
}));
|
||||
|
||||
let payload = serde_json::to_value(&event).expect("serialize responses api call event");
|
||||
|
||||
assert_eq!(payload["event_type"], "codex_responses_api_call_event");
|
||||
assert_eq!(payload["event_params"]["thread_id"], "thread-1");
|
||||
assert_eq!(payload["event_params"]["turn_id"], "turn-1");
|
||||
assert_eq!(payload["event_params"]["ephemeral"], false);
|
||||
assert_eq!(payload["event_params"]["thread_source"], "user");
|
||||
assert_eq!(payload["event_params"]["initialization_mode"], "new");
|
||||
assert_eq!(
|
||||
payload["event_params"]["app_server_client"]["product_client_id"],
|
||||
DEFAULT_ORIGINATOR
|
||||
);
|
||||
assert_eq!(
|
||||
payload["event_params"]["app_server_client"]["rpc_transport"],
|
||||
"stdio"
|
||||
);
|
||||
assert_eq!(
|
||||
payload["event_params"]["runtime"]["codex_rs_version"],
|
||||
"0.1.0"
|
||||
);
|
||||
assert_eq!(payload["event_params"]["responses_id"], "resp_123");
|
||||
assert_eq!(payload["event_params"]["turn_responses_call_index"], 2);
|
||||
assert_eq!(payload["event_params"]["model"], "gpt-5");
|
||||
assert_eq!(payload["event_params"]["model_provider"], "openai");
|
||||
assert_eq!(payload["event_params"]["reasoning_effort"], "high");
|
||||
assert_eq!(payload["event_params"]["status"], "completed");
|
||||
assert_eq!(payload["event_params"]["error"], json!(null));
|
||||
assert_eq!(payload["event_params"]["started_at"], 100);
|
||||
assert_eq!(payload["event_params"]["completed_at"], 102);
|
||||
assert_eq!(payload["event_params"]["duration_ms"], 2345);
|
||||
assert_eq!(payload["event_params"]["input_item_count"], 1);
|
||||
assert_eq!(payload["event_params"]["output_item_count"], 1);
|
||||
assert_eq!(payload["event_params"]["input_tokens"], 10);
|
||||
assert_eq!(payload["event_params"]["cached_input_tokens"], 4);
|
||||
assert_eq!(payload["event_params"]["output_tokens"], 20);
|
||||
assert_eq!(payload["event_params"]["reasoning_output_tokens"], 3);
|
||||
assert_eq!(payload["event_params"]["total_tokens"], 30);
|
||||
assert_eq!(payload["event_params"]["items"][0]["item_phase"], "input");
|
||||
assert_eq!(
|
||||
payload["event_params"]["items"][0]["response_item_type"],
|
||||
"message"
|
||||
);
|
||||
assert_eq!(payload["event_params"]["items"][0]["role"], "assistant");
|
||||
assert_eq!(
|
||||
payload["event_params"]["items"][0]["message_phase"],
|
||||
"final_answer"
|
||||
);
|
||||
assert_eq!(payload["event_params"]["items"][0]["text_part_count"], 1);
|
||||
assert_eq!(payload["event_params"]["items"][0]["image_part_count"], 0);
|
||||
assert_eq!(payload["event_params"]["items"][1]["item_phase"], "output");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn app_used_dedupe_is_keyed_by_turn_and_connector() {
|
||||
let (sender, _receiver) = mpsc::channel(1);
|
||||
@@ -1186,6 +1309,88 @@ async fn guardian_review_event_ingests_custom_fact_with_optional_target_item() {
|
||||
assert_eq!(payload[0]["event_params"]["review_timeout_ms"], 90_000);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn responses_api_call_event_ingests_custom_fact() {
|
||||
let mut reducer = AnalyticsReducer::default();
|
||||
let mut events = Vec::new();
|
||||
|
||||
ingest_turn_prerequisites(
|
||||
&mut reducer,
|
||||
&mut events,
|
||||
/*include_initialize*/ true,
|
||||
/*include_resolved_config*/ true,
|
||||
/*include_started*/ false,
|
||||
/*include_token_usage*/ false,
|
||||
)
|
||||
.await;
|
||||
events.clear();
|
||||
|
||||
reducer
|
||||
.ingest(
|
||||
AnalyticsFact::Custom(CustomAnalyticsFact::ResponsesApiCall(Box::new(
|
||||
CodexResponsesApiCallFact {
|
||||
thread_id: "thread-2".to_string(),
|
||||
turn_id: "turn-2".to_string(),
|
||||
responses_id: Some("resp_456".to_string()),
|
||||
turn_responses_call_index: 1,
|
||||
status: CodexResponsesApiCallStatus::Completed,
|
||||
error: None,
|
||||
started_at: 100,
|
||||
completed_at: Some(101),
|
||||
duration_ms: Some(1200),
|
||||
input_item_count: 1,
|
||||
output_item_count: 1,
|
||||
input_tokens: Some(123),
|
||||
cached_input_tokens: Some(45),
|
||||
output_tokens: Some(140),
|
||||
reasoning_output_tokens: Some(13),
|
||||
total_tokens: Some(321),
|
||||
items: vec![
|
||||
sample_responses_api_item_metadata(
|
||||
CodexResponsesApiItemPhase::Input,
|
||||
/*item_index*/ 0,
|
||||
),
|
||||
sample_responses_api_item_metadata(
|
||||
CodexResponsesApiItemPhase::Output,
|
||||
/*item_index*/ 0,
|
||||
),
|
||||
],
|
||||
},
|
||||
))),
|
||||
&mut events,
|
||||
)
|
||||
.await;
|
||||
|
||||
let payload = serde_json::to_value(&events).expect("serialize events");
|
||||
assert_eq!(payload.as_array().expect("events array").len(), 1);
|
||||
assert_eq!(payload[0]["event_type"], "codex_responses_api_call_event");
|
||||
assert_eq!(payload[0]["event_params"]["thread_id"], "thread-2");
|
||||
assert_eq!(payload[0]["event_params"]["turn_id"], "turn-2");
|
||||
assert_eq!(payload[0]["event_params"]["ephemeral"], false);
|
||||
assert_eq!(payload[0]["event_params"]["thread_source"], "user");
|
||||
assert_eq!(payload[0]["event_params"]["initialization_mode"], "new");
|
||||
assert_eq!(
|
||||
payload[0]["event_params"]["app_server_client"]["product_client_id"],
|
||||
"codex-tui"
|
||||
);
|
||||
assert_eq!(payload[0]["event_params"]["runtime"]["runtime_os"], "macos");
|
||||
assert_eq!(payload[0]["event_params"]["responses_id"], "resp_456");
|
||||
assert_eq!(payload[0]["event_params"]["turn_responses_call_index"], 1);
|
||||
assert_eq!(payload[0]["event_params"]["model"], "gpt-5");
|
||||
assert_eq!(payload[0]["event_params"]["model_provider"], "openai");
|
||||
assert_eq!(payload[0]["event_params"]["reasoning_effort"], json!(null));
|
||||
assert_eq!(payload[0]["event_params"]["status"], "completed");
|
||||
assert_eq!(payload[0]["event_params"]["input_tokens"], 123);
|
||||
assert_eq!(
|
||||
payload[0]["event_params"]["items"][0]["item_phase"],
|
||||
"input"
|
||||
);
|
||||
assert_eq!(
|
||||
payload[0]["event_params"]["items"][1]["item_phase"],
|
||||
"output"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn subagent_thread_started_review_serializes_expected_shape() {
|
||||
let event = TrackEventRequest::ThreadInitialized(subagent_thread_started_event_request(
|
||||
|
||||
@@ -8,6 +8,8 @@ use crate::facts::AnalyticsJsonRpcError;
|
||||
use crate::facts::AppInvocation;
|
||||
use crate::facts::AppMentionedInput;
|
||||
use crate::facts::AppUsedInput;
|
||||
use crate::facts::CodexResponsesApiCallFact;
|
||||
use crate::facts::CodexResponsesApiCallInput;
|
||||
use crate::facts::CustomAnalyticsFact;
|
||||
use crate::facts::HookRunFact;
|
||||
use crate::facts::HookRunInput;
|
||||
@@ -38,6 +40,7 @@ use tokio::sync::mpsc;
|
||||
const ANALYTICS_EVENTS_QUEUE_SIZE: usize = 256;
|
||||
const ANALYTICS_EVENTS_TIMEOUT: Duration = Duration::from_secs(10);
|
||||
const ANALYTICS_EVENT_DEDUPE_MAX_KEYS: usize = 4096;
|
||||
const RESPONSES_API_ERROR_MAX_BYTES: usize = 1024;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct AnalyticsEventsQueue {
|
||||
@@ -214,6 +217,40 @@ impl AnalyticsEventsClient {
|
||||
)));
|
||||
}
|
||||
|
||||
pub fn track_responses_api_call(
|
||||
&self,
|
||||
tracking: TrackEventsContext,
|
||||
input: CodexResponsesApiCallInput,
|
||||
) {
|
||||
let token_usage = input.token_usage;
|
||||
let error = input.error.map(truncate_responses_api_error);
|
||||
let event = CodexResponsesApiCallFact {
|
||||
thread_id: tracking.thread_id,
|
||||
turn_id: tracking.turn_id,
|
||||
responses_id: input.responses_id,
|
||||
turn_responses_call_index: input.turn_responses_call_index,
|
||||
status: input.status,
|
||||
error,
|
||||
started_at: input.started_at,
|
||||
completed_at: input.completed_at,
|
||||
duration_ms: input.duration_ms,
|
||||
input_item_count: input.input_item_count,
|
||||
output_item_count: input.output_item_count,
|
||||
input_tokens: token_usage.as_ref().map(|usage| usage.input_tokens),
|
||||
cached_input_tokens: token_usage.as_ref().map(|usage| usage.cached_input_tokens),
|
||||
output_tokens: token_usage.as_ref().map(|usage| usage.output_tokens),
|
||||
reasoning_output_tokens: token_usage
|
||||
.as_ref()
|
||||
.map(|usage| usage.reasoning_output_tokens),
|
||||
total_tokens: token_usage.as_ref().map(|usage| usage.total_tokens),
|
||||
items: input.items,
|
||||
};
|
||||
|
||||
self.record_fact(AnalyticsFact::Custom(
|
||||
CustomAnalyticsFact::ResponsesApiCall(Box::new(event)),
|
||||
));
|
||||
}
|
||||
|
||||
pub fn track_turn_resolved_config(&self, fact: TurnResolvedConfigFact) {
|
||||
self.record_fact(AnalyticsFact::Custom(
|
||||
CustomAnalyticsFact::TurnResolvedConfig(Box::new(fact)),
|
||||
@@ -296,6 +333,18 @@ impl AnalyticsEventsClient {
|
||||
}
|
||||
}
|
||||
|
||||
fn truncate_responses_api_error(mut error: String) -> String {
|
||||
if error.len() <= RESPONSES_API_ERROR_MAX_BYTES {
|
||||
return error;
|
||||
}
|
||||
let mut truncate_at = RESPONSES_API_ERROR_MAX_BYTES;
|
||||
while !error.is_char_boundary(truncate_at) {
|
||||
truncate_at -= 1;
|
||||
}
|
||||
error.truncate(truncate_at);
|
||||
error
|
||||
}
|
||||
|
||||
async fn send_track_events(
|
||||
auth_manager: &Arc<AuthManager>,
|
||||
base_url: &str,
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
use crate::facts::AppInvocation;
|
||||
use crate::facts::CodexCompactionEvent;
|
||||
use crate::facts::CodexResponsesApiCallStatus;
|
||||
use crate::facts::CodexResponsesApiItemMetadata;
|
||||
use crate::facts::CompactionImplementation;
|
||||
use crate::facts::CompactionPhase;
|
||||
use crate::facts::CompactionReason;
|
||||
@@ -55,6 +57,7 @@ pub(crate) enum TrackEventRequest {
|
||||
AppUsed(CodexAppUsedEventRequest),
|
||||
HookRun(CodexHookRunEventRequest),
|
||||
Compaction(Box<CodexCompactionEventRequest>),
|
||||
ResponsesApiCall(Box<CodexResponsesApiCallEventRequest>),
|
||||
TurnEvent(Box<CodexTurnEventRequest>),
|
||||
TurnSteer(CodexTurnSteerEventRequest),
|
||||
PluginUsed(CodexPluginUsedEventRequest),
|
||||
@@ -311,6 +314,43 @@ pub(crate) struct CodexCompactionEventRequest {
|
||||
pub(crate) event_params: CodexCompactionEventParams,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub(crate) struct CodexResponsesApiCallEventParams {
|
||||
pub(crate) thread_id: String,
|
||||
pub(crate) turn_id: String,
|
||||
pub(crate) ephemeral: bool,
|
||||
pub(crate) thread_source: Option<String>,
|
||||
pub(crate) initialization_mode: ThreadInitializationMode,
|
||||
pub(crate) subagent_source: Option<String>,
|
||||
pub(crate) parent_thread_id: Option<String>,
|
||||
pub(crate) app_server_client: CodexAppServerClientMetadata,
|
||||
pub(crate) runtime: CodexRuntimeMetadata,
|
||||
pub(crate) responses_id: Option<String>,
|
||||
pub(crate) turn_responses_call_index: u64,
|
||||
pub(crate) model: Option<String>,
|
||||
pub(crate) model_provider: Option<String>,
|
||||
pub(crate) reasoning_effort: Option<String>,
|
||||
pub(crate) status: CodexResponsesApiCallStatus,
|
||||
pub(crate) error: Option<String>,
|
||||
pub(crate) started_at: u64,
|
||||
pub(crate) completed_at: Option<u64>,
|
||||
pub(crate) duration_ms: Option<u64>,
|
||||
pub(crate) input_item_count: usize,
|
||||
pub(crate) output_item_count: usize,
|
||||
pub(crate) input_tokens: Option<i64>,
|
||||
pub(crate) cached_input_tokens: Option<i64>,
|
||||
pub(crate) output_tokens: Option<i64>,
|
||||
pub(crate) reasoning_output_tokens: Option<i64>,
|
||||
pub(crate) total_tokens: Option<i64>,
|
||||
pub(crate) items: Vec<CodexResponsesApiItemMetadata>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub(crate) struct CodexResponsesApiCallEventRequest {
|
||||
pub(crate) event_type: &'static str,
|
||||
pub(crate) event_params: CodexResponsesApiCallEventParams,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub(crate) struct CodexTurnEventParams {
|
||||
pub(crate) thread_id: String,
|
||||
|
||||
@@ -13,6 +13,7 @@ use codex_protocol::config_types::ModeKind;
|
||||
use codex_protocol::config_types::Personality;
|
||||
use codex_protocol::config_types::ReasoningSummary;
|
||||
use codex_protocol::config_types::ServiceTier;
|
||||
use codex_protocol::models::MessagePhase;
|
||||
use codex_protocol::openai_models::ReasoningEffort;
|
||||
use codex_protocol::protocol::AskForApproval;
|
||||
use codex_protocol::protocol::HookEventName;
|
||||
@@ -262,6 +263,91 @@ pub struct CodexCompactionEvent {
|
||||
pub duration_ms: Option<u64>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Serialize, PartialEq, Eq)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum CodexResponsesApiCallStatus {
|
||||
Completed,
|
||||
Failed,
|
||||
Interrupted,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Serialize, PartialEq, Eq)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum CodexResponsesApiItemPhase {
|
||||
Input,
|
||||
Output,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Serialize, PartialEq, Eq)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum CodexResponseItemType {
|
||||
Message,
|
||||
Reasoning,
|
||||
LocalShellCall,
|
||||
FunctionCall,
|
||||
FunctionCallOutput,
|
||||
CustomToolCall,
|
||||
CustomToolCallOutput,
|
||||
ToolSearchCall,
|
||||
ToolSearchOutput,
|
||||
WebSearchCall,
|
||||
ImageGenerationCall,
|
||||
GhostSnapshot,
|
||||
Compaction,
|
||||
Other,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize, PartialEq, Eq)]
|
||||
pub struct CodexResponsesApiItemMetadata {
|
||||
pub item_phase: CodexResponsesApiItemPhase,
|
||||
pub item_index: usize,
|
||||
pub response_item_type: CodexResponseItemType,
|
||||
pub role: Option<String>,
|
||||
pub status: Option<String>,
|
||||
pub message_phase: Option<MessagePhase>,
|
||||
pub call_id: Option<String>,
|
||||
pub tool_name: Option<String>,
|
||||
pub payload_bytes: Option<i64>,
|
||||
pub text_part_count: Option<usize>,
|
||||
pub image_part_count: Option<usize>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct CodexResponsesApiCallInput {
|
||||
pub responses_id: Option<String>,
|
||||
pub turn_responses_call_index: u64,
|
||||
pub status: CodexResponsesApiCallStatus,
|
||||
pub error: Option<String>,
|
||||
pub started_at: u64,
|
||||
pub completed_at: Option<u64>,
|
||||
pub duration_ms: Option<u64>,
|
||||
pub input_item_count: usize,
|
||||
pub output_item_count: usize,
|
||||
pub items: Vec<CodexResponsesApiItemMetadata>,
|
||||
pub token_usage: Option<TokenUsage>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct CodexResponsesApiCallFact {
|
||||
pub thread_id: String,
|
||||
pub turn_id: String,
|
||||
pub responses_id: Option<String>,
|
||||
pub turn_responses_call_index: u64,
|
||||
pub status: CodexResponsesApiCallStatus,
|
||||
pub error: Option<String>,
|
||||
pub started_at: u64,
|
||||
pub completed_at: Option<u64>,
|
||||
pub duration_ms: Option<u64>,
|
||||
pub input_item_count: usize,
|
||||
pub output_item_count: usize,
|
||||
pub input_tokens: Option<i64>,
|
||||
pub cached_input_tokens: Option<i64>,
|
||||
pub output_tokens: Option<i64>,
|
||||
pub reasoning_output_tokens: Option<i64>,
|
||||
pub total_tokens: Option<i64>,
|
||||
pub items: Vec<CodexResponsesApiItemMetadata>,
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
pub(crate) enum AnalyticsFact {
|
||||
Initialize {
|
||||
@@ -295,6 +381,7 @@ pub(crate) enum AnalyticsFact {
|
||||
pub(crate) enum CustomAnalyticsFact {
|
||||
SubAgentThreadStarted(SubAgentThreadStartedInput),
|
||||
Compaction(Box<CodexCompactionEvent>),
|
||||
ResponsesApiCall(Box<CodexResponsesApiCallFact>),
|
||||
GuardianReview(Box<GuardianReviewEventParams>),
|
||||
TurnResolvedConfig(Box<TurnResolvedConfigFact>),
|
||||
TurnTokenUsage(Box<TurnTokenUsageFact>),
|
||||
|
||||
@@ -2,7 +2,10 @@ mod client;
|
||||
mod events;
|
||||
mod facts;
|
||||
mod reducer;
|
||||
mod response_items;
|
||||
|
||||
use codex_protocol::models::ResponseItem;
|
||||
use serde::Serialize;
|
||||
use std::time::SystemTime;
|
||||
use std::time::UNIX_EPOCH;
|
||||
|
||||
@@ -18,6 +21,9 @@ pub use events::GuardianReviewedAction;
|
||||
pub use facts::AnalyticsJsonRpcError;
|
||||
pub use facts::AppInvocation;
|
||||
pub use facts::CodexCompactionEvent;
|
||||
pub use facts::CodexResponsesApiCallInput;
|
||||
pub use facts::CodexResponsesApiCallStatus;
|
||||
pub use facts::CodexResponsesApiItemMetadata;
|
||||
pub use facts::CodexTurnSteerEvent;
|
||||
pub use facts::CompactionImplementation;
|
||||
pub use facts::CompactionPhase;
|
||||
@@ -49,3 +55,41 @@ pub fn now_unix_seconds() -> u64 {
|
||||
.unwrap_or_default()
|
||||
.as_secs()
|
||||
}
|
||||
|
||||
pub fn responses_api_input_items_metadata(
|
||||
items: &[ResponseItem],
|
||||
) -> Vec<CodexResponsesApiItemMetadata> {
|
||||
response_items::response_items_metadata(facts::CodexResponsesApiItemPhase::Input, items)
|
||||
}
|
||||
|
||||
pub fn responses_api_output_item_metadata(
|
||||
item_index: usize,
|
||||
item: &ResponseItem,
|
||||
) -> CodexResponsesApiItemMetadata {
|
||||
response_items::response_item_metadata(
|
||||
facts::CodexResponsesApiItemPhase::Output,
|
||||
item_index,
|
||||
item,
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) fn serialized_string<T: Serialize>(value: &T) -> Option<String> {
|
||||
match serde_json::to_value(value).ok()? {
|
||||
serde_json::Value::String(value) => Some(value),
|
||||
value => Some(value.to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn serialized_bytes<T: Serialize>(value: &T) -> Option<i64> {
|
||||
serde_json::to_string(value)
|
||||
.ok()
|
||||
.map(|value| byte_len(&value))
|
||||
}
|
||||
|
||||
pub(crate) fn nonzero_i64(value: i64) -> Option<i64> {
|
||||
(value > 0).then_some(value)
|
||||
}
|
||||
|
||||
pub(crate) fn byte_len(value: &str) -> i64 {
|
||||
i64::try_from(value.len()).unwrap_or(i64::MAX)
|
||||
}
|
||||
|
||||
@@ -6,6 +6,8 @@ use crate::events::CodexCompactionEventRequest;
|
||||
use crate::events::CodexHookRunEventRequest;
|
||||
use crate::events::CodexPluginEventRequest;
|
||||
use crate::events::CodexPluginUsedEventRequest;
|
||||
use crate::events::CodexResponsesApiCallEventParams;
|
||||
use crate::events::CodexResponsesApiCallEventRequest;
|
||||
use crate::events::CodexRuntimeMetadata;
|
||||
use crate::events::CodexTurnEventParams;
|
||||
use crate::events::CodexTurnEventRequest;
|
||||
@@ -33,6 +35,7 @@ use crate::facts::AnalyticsJsonRpcError;
|
||||
use crate::facts::AppMentionedInput;
|
||||
use crate::facts::AppUsedInput;
|
||||
use crate::facts::CodexCompactionEvent;
|
||||
use crate::facts::CodexResponsesApiCallFact;
|
||||
use crate::facts::CustomAnalyticsFact;
|
||||
use crate::facts::HookRunInput;
|
||||
use crate::facts::PluginState;
|
||||
@@ -202,6 +205,9 @@ impl AnalyticsReducer {
|
||||
CustomAnalyticsFact::Compaction(input) => {
|
||||
self.ingest_compaction(*input, out);
|
||||
}
|
||||
CustomAnalyticsFact::ResponsesApiCall(input) => {
|
||||
self.ingest_responses_api_call(*input, out);
|
||||
}
|
||||
CustomAnalyticsFact::GuardianReview(input) => {
|
||||
self.ingest_guardian_review(*input, out);
|
||||
}
|
||||
@@ -739,6 +745,91 @@ impl AnalyticsReducer {
|
||||
)));
|
||||
}
|
||||
|
||||
fn ingest_responses_api_call(
|
||||
&mut self,
|
||||
input: CodexResponsesApiCallFact,
|
||||
out: &mut Vec<TrackEventRequest>,
|
||||
) {
|
||||
let Some(connection_id) = self.thread_connections.get(&input.thread_id) else {
|
||||
tracing::warn!(
|
||||
thread_id = %input.thread_id,
|
||||
turn_id = %input.turn_id,
|
||||
"dropping responses api call analytics event: missing thread connection metadata"
|
||||
);
|
||||
return;
|
||||
};
|
||||
let Some(connection_state) = self.connections.get(connection_id) else {
|
||||
tracing::warn!(
|
||||
thread_id = %input.thread_id,
|
||||
turn_id = %input.turn_id,
|
||||
connection_id,
|
||||
"dropping responses api call analytics event: missing connection metadata"
|
||||
);
|
||||
return;
|
||||
};
|
||||
let Some(thread_metadata) = self.thread_metadata.get(&input.thread_id) else {
|
||||
tracing::warn!(
|
||||
thread_id = %input.thread_id,
|
||||
turn_id = %input.turn_id,
|
||||
"dropping responses api call analytics event: missing thread lifecycle metadata"
|
||||
);
|
||||
return;
|
||||
};
|
||||
let Some(turn_state) = self.turns.get(&input.turn_id) else {
|
||||
tracing::warn!(
|
||||
thread_id = %input.thread_id,
|
||||
turn_id = %input.turn_id,
|
||||
"dropping responses api call analytics event: missing turn metadata"
|
||||
);
|
||||
return;
|
||||
};
|
||||
let Some(resolved_config) = turn_state.resolved_config.as_ref() else {
|
||||
tracing::warn!(
|
||||
thread_id = %input.thread_id,
|
||||
turn_id = %input.turn_id,
|
||||
"dropping responses api call analytics event: missing turn resolved config"
|
||||
);
|
||||
return;
|
||||
};
|
||||
|
||||
out.push(TrackEventRequest::ResponsesApiCall(Box::new(
|
||||
CodexResponsesApiCallEventRequest {
|
||||
event_type: "codex_responses_api_call_event",
|
||||
event_params: CodexResponsesApiCallEventParams {
|
||||
thread_id: input.thread_id,
|
||||
turn_id: input.turn_id,
|
||||
ephemeral: resolved_config.ephemeral,
|
||||
thread_source: thread_metadata.thread_source.map(str::to_string),
|
||||
initialization_mode: thread_metadata.initialization_mode,
|
||||
subagent_source: thread_metadata.subagent_source.clone(),
|
||||
parent_thread_id: thread_metadata.parent_thread_id.clone(),
|
||||
app_server_client: connection_state.app_server_client.clone(),
|
||||
runtime: connection_state.runtime.clone(),
|
||||
responses_id: input.responses_id,
|
||||
turn_responses_call_index: input.turn_responses_call_index,
|
||||
model: Some(resolved_config.model.clone()),
|
||||
model_provider: Some(resolved_config.model_provider.clone()),
|
||||
reasoning_effort: resolved_config
|
||||
.reasoning_effort
|
||||
.map(|value| value.to_string()),
|
||||
status: input.status,
|
||||
error: input.error,
|
||||
started_at: input.started_at,
|
||||
completed_at: input.completed_at,
|
||||
duration_ms: input.duration_ms,
|
||||
input_item_count: input.input_item_count,
|
||||
output_item_count: input.output_item_count,
|
||||
input_tokens: input.input_tokens,
|
||||
cached_input_tokens: input.cached_input_tokens,
|
||||
output_tokens: input.output_tokens,
|
||||
reasoning_output_tokens: input.reasoning_output_tokens,
|
||||
total_tokens: input.total_tokens,
|
||||
items: input.items,
|
||||
},
|
||||
},
|
||||
)));
|
||||
}
|
||||
|
||||
fn ingest_turn_steer_response(
|
||||
&mut self,
|
||||
connection_id: u64,
|
||||
|
||||
366
codex-rs/analytics/src/response_items.rs
Normal file
366
codex-rs/analytics/src/response_items.rs
Normal file
@@ -0,0 +1,366 @@
|
||||
use crate::byte_len;
|
||||
use crate::facts::CodexResponseItemType;
|
||||
use crate::facts::CodexResponsesApiItemMetadata;
|
||||
use crate::facts::CodexResponsesApiItemPhase;
|
||||
use crate::nonzero_i64;
|
||||
use crate::serialized_bytes;
|
||||
use crate::serialized_string;
|
||||
use codex_protocol::models::ContentItem;
|
||||
use codex_protocol::models::FunctionCallOutputBody;
|
||||
use codex_protocol::models::FunctionCallOutputContentItem;
|
||||
use codex_protocol::models::FunctionCallOutputPayload;
|
||||
use codex_protocol::models::ReasoningItemContent;
|
||||
use codex_protocol::models::ReasoningItemReasoningSummary;
|
||||
use codex_protocol::models::ResponseItem;
|
||||
|
||||
pub(crate) fn response_items_metadata(
|
||||
phase: CodexResponsesApiItemPhase,
|
||||
items: &[ResponseItem],
|
||||
) -> Vec<CodexResponsesApiItemMetadata> {
|
||||
items
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(item_index, item)| response_item_metadata(phase, item_index, item))
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub(crate) fn response_item_metadata(
|
||||
item_phase: CodexResponsesApiItemPhase,
|
||||
item_index: usize,
|
||||
item: &ResponseItem,
|
||||
) -> CodexResponsesApiItemMetadata {
|
||||
let mut metadata = new_metadata(item_phase, item_index, response_item_type(item));
|
||||
|
||||
match item {
|
||||
ResponseItem::Message {
|
||||
role,
|
||||
content,
|
||||
phase,
|
||||
..
|
||||
} => {
|
||||
metadata.role = Some(role.clone());
|
||||
metadata.message_phase = phase.clone();
|
||||
metadata.payload_bytes = nonzero_i64(message_content_text_bytes(content));
|
||||
let (text_part_count, image_part_count) = message_content_part_counts(content);
|
||||
metadata.text_part_count = Some(text_part_count);
|
||||
metadata.image_part_count = Some(image_part_count);
|
||||
}
|
||||
ResponseItem::Reasoning {
|
||||
summary,
|
||||
content,
|
||||
encrypted_content,
|
||||
..
|
||||
} => {
|
||||
metadata.payload_bytes = encrypted_content
|
||||
.as_ref()
|
||||
.map(|value| byte_len(value))
|
||||
.or_else(|| nonzero_i64(reasoning_content_bytes(summary, content)));
|
||||
metadata.text_part_count =
|
||||
Some(summary.len() + content.as_ref().map(std::vec::Vec::len).unwrap_or_default());
|
||||
metadata.image_part_count = Some(0);
|
||||
}
|
||||
ResponseItem::LocalShellCall {
|
||||
call_id,
|
||||
status,
|
||||
action,
|
||||
..
|
||||
} => {
|
||||
metadata.call_id = call_id.clone();
|
||||
metadata.tool_name = Some("local_shell".to_string());
|
||||
metadata.status = serialized_string(status);
|
||||
metadata.payload_bytes = serialized_bytes(action);
|
||||
}
|
||||
ResponseItem::FunctionCall {
|
||||
name,
|
||||
arguments,
|
||||
call_id,
|
||||
..
|
||||
} => {
|
||||
metadata.call_id = Some(call_id.clone());
|
||||
metadata.tool_name = Some(name.clone());
|
||||
metadata.payload_bytes = Some(byte_len(arguments));
|
||||
}
|
||||
ResponseItem::ToolSearchCall {
|
||||
call_id,
|
||||
status,
|
||||
arguments,
|
||||
..
|
||||
} => {
|
||||
metadata.call_id = call_id.clone();
|
||||
metadata.tool_name = Some("tool_search".to_string());
|
||||
metadata.status = status.clone();
|
||||
metadata.payload_bytes = serialized_bytes(arguments);
|
||||
}
|
||||
ResponseItem::FunctionCallOutput { call_id, output } => {
|
||||
metadata.call_id = Some(call_id.clone());
|
||||
metadata.payload_bytes = function_call_output_bytes(output);
|
||||
let (text_part_count, image_part_count) = function_call_output_part_counts(output);
|
||||
metadata.text_part_count = text_part_count;
|
||||
metadata.image_part_count = image_part_count;
|
||||
}
|
||||
ResponseItem::CustomToolCall {
|
||||
status,
|
||||
call_id,
|
||||
name,
|
||||
input,
|
||||
..
|
||||
} => {
|
||||
metadata.call_id = Some(call_id.clone());
|
||||
metadata.tool_name = Some(name.clone());
|
||||
metadata.status = status.clone();
|
||||
metadata.payload_bytes = Some(byte_len(input));
|
||||
}
|
||||
ResponseItem::CustomToolCallOutput {
|
||||
call_id,
|
||||
name,
|
||||
output,
|
||||
} => {
|
||||
metadata.call_id = Some(call_id.clone());
|
||||
metadata.tool_name = name.clone();
|
||||
metadata.payload_bytes = function_call_output_bytes(output);
|
||||
let (text_part_count, image_part_count) = function_call_output_part_counts(output);
|
||||
metadata.text_part_count = text_part_count;
|
||||
metadata.image_part_count = image_part_count;
|
||||
}
|
||||
ResponseItem::ToolSearchOutput {
|
||||
call_id,
|
||||
status,
|
||||
tools,
|
||||
..
|
||||
} => {
|
||||
metadata.call_id = call_id.clone();
|
||||
metadata.tool_name = Some("tool_search".to_string());
|
||||
metadata.status = Some(status.clone());
|
||||
metadata.payload_bytes = serialized_bytes(tools);
|
||||
}
|
||||
ResponseItem::WebSearchCall { status, action, .. } => {
|
||||
metadata.tool_name = Some("web_search".to_string());
|
||||
metadata.status = status.clone();
|
||||
metadata.payload_bytes = action.as_ref().and_then(serialized_bytes);
|
||||
}
|
||||
ResponseItem::ImageGenerationCall {
|
||||
id,
|
||||
status,
|
||||
revised_prompt,
|
||||
result,
|
||||
} => {
|
||||
metadata.call_id = Some(id.clone());
|
||||
metadata.tool_name = Some("image_generation".to_string());
|
||||
metadata.status = Some(status.clone());
|
||||
metadata.payload_bytes = nonzero_i64(byte_len(result))
|
||||
.or_else(|| revised_prompt.as_ref().map(|value| byte_len(value)));
|
||||
}
|
||||
ResponseItem::Compaction { encrypted_content } => {
|
||||
metadata.payload_bytes = Some(byte_len(encrypted_content));
|
||||
}
|
||||
ResponseItem::GhostSnapshot { .. } | ResponseItem::Other => {}
|
||||
}
|
||||
|
||||
metadata
|
||||
}
|
||||
|
||||
fn new_metadata(
|
||||
item_phase: CodexResponsesApiItemPhase,
|
||||
item_index: usize,
|
||||
response_item_type: CodexResponseItemType,
|
||||
) -> CodexResponsesApiItemMetadata {
|
||||
CodexResponsesApiItemMetadata {
|
||||
item_phase,
|
||||
item_index,
|
||||
response_item_type,
|
||||
role: None,
|
||||
status: None,
|
||||
message_phase: None,
|
||||
call_id: None,
|
||||
tool_name: None,
|
||||
payload_bytes: None,
|
||||
text_part_count: None,
|
||||
image_part_count: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn response_item_type(item: &ResponseItem) -> CodexResponseItemType {
|
||||
match item {
|
||||
ResponseItem::Message { .. } => CodexResponseItemType::Message,
|
||||
ResponseItem::Reasoning { .. } => CodexResponseItemType::Reasoning,
|
||||
ResponseItem::LocalShellCall { .. } => CodexResponseItemType::LocalShellCall,
|
||||
ResponseItem::FunctionCall { .. } => CodexResponseItemType::FunctionCall,
|
||||
ResponseItem::ToolSearchCall { .. } => CodexResponseItemType::ToolSearchCall,
|
||||
ResponseItem::FunctionCallOutput { .. } => CodexResponseItemType::FunctionCallOutput,
|
||||
ResponseItem::CustomToolCall { .. } => CodexResponseItemType::CustomToolCall,
|
||||
ResponseItem::CustomToolCallOutput { .. } => CodexResponseItemType::CustomToolCallOutput,
|
||||
ResponseItem::ToolSearchOutput { .. } => CodexResponseItemType::ToolSearchOutput,
|
||||
ResponseItem::WebSearchCall { .. } => CodexResponseItemType::WebSearchCall,
|
||||
ResponseItem::ImageGenerationCall { .. } => CodexResponseItemType::ImageGenerationCall,
|
||||
ResponseItem::GhostSnapshot { .. } => CodexResponseItemType::GhostSnapshot,
|
||||
ResponseItem::Compaction { .. } => CodexResponseItemType::Compaction,
|
||||
ResponseItem::Other => CodexResponseItemType::Other,
|
||||
}
|
||||
}
|
||||
|
||||
fn message_content_text_bytes(content: &[ContentItem]) -> i64 {
|
||||
content
|
||||
.iter()
|
||||
.map(|item| match item {
|
||||
ContentItem::InputText { text } | ContentItem::OutputText { text } => byte_len(text),
|
||||
ContentItem::InputImage { .. } => 0,
|
||||
})
|
||||
.sum()
|
||||
}
|
||||
|
||||
fn message_content_part_counts(content: &[ContentItem]) -> (usize, usize) {
|
||||
let mut text_part_count = 0;
|
||||
let mut image_part_count = 0;
|
||||
for item in content {
|
||||
match item {
|
||||
ContentItem::InputText { .. } | ContentItem::OutputText { .. } => {
|
||||
text_part_count += 1;
|
||||
}
|
||||
ContentItem::InputImage { .. } => {
|
||||
image_part_count += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
(text_part_count, image_part_count)
|
||||
}
|
||||
|
||||
fn reasoning_content_bytes(
|
||||
summary: &[ReasoningItemReasoningSummary],
|
||||
content: &Option<Vec<ReasoningItemContent>>,
|
||||
) -> i64 {
|
||||
let summary_bytes = summary
|
||||
.iter()
|
||||
.map(|summary| match summary {
|
||||
ReasoningItemReasoningSummary::SummaryText { text } => byte_len(text),
|
||||
})
|
||||
.sum::<i64>();
|
||||
let content_bytes = content
|
||||
.as_ref()
|
||||
.map(|content| {
|
||||
content
|
||||
.iter()
|
||||
.map(|content| match content {
|
||||
ReasoningItemContent::ReasoningText { text }
|
||||
| ReasoningItemContent::Text { text } => byte_len(text),
|
||||
})
|
||||
.sum::<i64>()
|
||||
})
|
||||
.unwrap_or_default();
|
||||
summary_bytes + content_bytes
|
||||
}
|
||||
|
||||
fn function_call_output_bytes(output: &FunctionCallOutputPayload) -> Option<i64> {
|
||||
match &output.body {
|
||||
FunctionCallOutputBody::Text(text) => Some(byte_len(text)),
|
||||
FunctionCallOutputBody::ContentItems(items) => serialized_bytes(items),
|
||||
}
|
||||
}
|
||||
|
||||
fn function_call_output_part_counts(
|
||||
output: &FunctionCallOutputPayload,
|
||||
) -> (Option<usize>, Option<usize>) {
|
||||
match &output.body {
|
||||
FunctionCallOutputBody::Text(_) => (Some(1), Some(0)),
|
||||
FunctionCallOutputBody::ContentItems(content_items) => {
|
||||
let mut text_part_count = 0;
|
||||
let mut image_part_count = 0;
|
||||
for item in content_items {
|
||||
match item {
|
||||
FunctionCallOutputContentItem::InputText { .. } => {
|
||||
text_part_count += 1;
|
||||
}
|
||||
FunctionCallOutputContentItem::InputImage { .. } => {
|
||||
image_part_count += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
(Some(text_part_count), Some(image_part_count))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use codex_protocol::models::ImageDetail;
|
||||
use codex_protocol::models::MessagePhase;
|
||||
|
||||
#[test]
|
||||
fn maps_message_metadata() {
|
||||
let items = vec![ResponseItem::Message {
|
||||
id: None,
|
||||
role: "assistant".to_string(),
|
||||
content: vec![
|
||||
ContentItem::OutputText {
|
||||
text: "hello".to_string(),
|
||||
},
|
||||
ContentItem::InputImage {
|
||||
image_url: "data:image/png;base64,abc".to_string(),
|
||||
detail: None,
|
||||
},
|
||||
],
|
||||
end_turn: None,
|
||||
phase: Some(MessagePhase::FinalAnswer),
|
||||
}];
|
||||
|
||||
let metadata = response_items_metadata(CodexResponsesApiItemPhase::Output, &items);
|
||||
|
||||
assert_eq!(metadata[0].item_phase, CodexResponsesApiItemPhase::Output);
|
||||
assert_eq!(
|
||||
metadata[0].response_item_type,
|
||||
CodexResponseItemType::Message
|
||||
);
|
||||
assert_eq!(metadata[0].role.as_deref(), Some("assistant"));
|
||||
assert_eq!(metadata[0].message_phase, Some(MessagePhase::FinalAnswer));
|
||||
assert_eq!(metadata[0].payload_bytes, Some(5));
|
||||
assert_eq!(metadata[0].text_part_count, Some(1));
|
||||
assert_eq!(metadata[0].image_part_count, Some(1));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn maps_tool_call_output_metadata() {
|
||||
let items = vec![ResponseItem::CustomToolCallOutput {
|
||||
call_id: "call_1".to_string(),
|
||||
name: Some("custom_tool".to_string()),
|
||||
output: FunctionCallOutputPayload::from_content_items(vec![
|
||||
FunctionCallOutputContentItem::InputText {
|
||||
text: "result".to_string(),
|
||||
},
|
||||
FunctionCallOutputContentItem::InputImage {
|
||||
image_url: "https://example.test/image.png".to_string(),
|
||||
detail: Some(ImageDetail::High),
|
||||
},
|
||||
]),
|
||||
}];
|
||||
|
||||
let metadata = response_items_metadata(CodexResponsesApiItemPhase::Output, &items);
|
||||
|
||||
assert_eq!(
|
||||
metadata[0].response_item_type,
|
||||
CodexResponseItemType::CustomToolCallOutput
|
||||
);
|
||||
assert_eq!(metadata[0].call_id.as_deref(), Some("call_1"));
|
||||
assert_eq!(metadata[0].tool_name.as_deref(), Some("custom_tool"));
|
||||
assert!(metadata[0].payload_bytes.unwrap_or_default() > 0);
|
||||
assert_eq!(metadata[0].text_part_count, Some(1));
|
||||
assert_eq!(metadata[0].image_part_count, Some(1));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn maps_text_tool_call_output_metadata() {
|
||||
let items = vec![ResponseItem::FunctionCallOutput {
|
||||
call_id: "call_1".to_string(),
|
||||
output: FunctionCallOutputPayload::from_text("result".to_string()),
|
||||
}];
|
||||
|
||||
let metadata = response_items_metadata(CodexResponsesApiItemPhase::Output, &items);
|
||||
|
||||
assert_eq!(
|
||||
metadata[0].response_item_type,
|
||||
CodexResponseItemType::FunctionCallOutput
|
||||
);
|
||||
assert_eq!(metadata[0].payload_bytes, Some(6));
|
||||
assert_eq!(metadata[0].text_part_count, Some(1));
|
||||
assert_eq!(metadata[0].image_part_count, Some(0));
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
use std::collections::HashMap;
|
||||
use std::collections::HashSet;
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
|
||||
use crate::SkillInjections;
|
||||
use crate::SkillLoadOutcome;
|
||||
@@ -59,11 +60,17 @@ use crate::unavailable_tool::collect_unavailable_called_tools;
|
||||
use crate::util::backoff;
|
||||
use crate::util::error_or_panic;
|
||||
use codex_analytics::AppInvocation;
|
||||
use codex_analytics::CodexResponsesApiCallInput;
|
||||
use codex_analytics::CodexResponsesApiCallStatus;
|
||||
use codex_analytics::CodexResponsesApiItemMetadata;
|
||||
use codex_analytics::CompactionPhase;
|
||||
use codex_analytics::CompactionReason;
|
||||
use codex_analytics::InvocationType;
|
||||
use codex_analytics::TurnResolvedConfigFact;
|
||||
use codex_analytics::build_track_events_context;
|
||||
use codex_analytics::now_unix_seconds;
|
||||
use codex_analytics::responses_api_input_items_metadata;
|
||||
use codex_analytics::responses_api_output_item_metadata;
|
||||
use codex_async_utils::OrCancelExt;
|
||||
use codex_features::Feature;
|
||||
use codex_hooks::HookEvent;
|
||||
@@ -91,6 +98,7 @@ use codex_protocol::protocol::EventMsg;
|
||||
use codex_protocol::protocol::PlanDeltaEvent;
|
||||
use codex_protocol::protocol::ReasoningContentDeltaEvent;
|
||||
use codex_protocol::protocol::ReasoningRawContentDeltaEvent;
|
||||
use codex_protocol::protocol::TokenUsage;
|
||||
use codex_protocol::protocol::TurnDiffEvent;
|
||||
use codex_protocol::protocol::WarningEvent;
|
||||
use codex_protocol::user_input::UserInput;
|
||||
@@ -406,6 +414,7 @@ pub(crate) async fn run_turn(
|
||||
// 1. At the start of a turn, so the fresh user prompt in `input` gets sampled first.
|
||||
// 2. After auto-compact, when model/tool continuation needs to resume before any steer.
|
||||
let mut can_drain_pending_input = input.is_empty();
|
||||
let mut next_turn_responses_call_index: u64 = 0;
|
||||
|
||||
loop {
|
||||
if run_pending_session_start_hooks(&sess, &turn_context).await {
|
||||
@@ -483,6 +492,7 @@ pub(crate) async fn run_turn(
|
||||
Arc::clone(&turn_diff_tracker),
|
||||
&mut client_session,
|
||||
turn_metadata_header.as_deref(),
|
||||
&mut next_turn_responses_call_index,
|
||||
sampling_request_input,
|
||||
&explicitly_enabled_connectors,
|
||||
skills_outcome,
|
||||
@@ -495,6 +505,7 @@ pub(crate) async fn run_turn(
|
||||
let SamplingRequestResult {
|
||||
needs_follow_up: model_needs_follow_up,
|
||||
last_agent_message: sampling_request_last_agent_message,
|
||||
..
|
||||
} = sampling_request_output;
|
||||
can_drain_pending_input = true;
|
||||
let has_pending_input = sess.has_pending_input().await;
|
||||
@@ -1032,6 +1043,81 @@ fn filter_deferred_dynamic_tool_spec(
|
||||
}
|
||||
}
|
||||
|
||||
struct ResponsesApiCallAttempt {
|
||||
turn_responses_call_index: u64,
|
||||
started_at: u64,
|
||||
started_instant: Instant,
|
||||
stream_started: bool,
|
||||
input_item_count: usize,
|
||||
output_item_count: usize,
|
||||
items: Vec<CodexResponsesApiItemMetadata>,
|
||||
responses_id: Option<String>,
|
||||
token_usage: Option<TokenUsage>,
|
||||
}
|
||||
|
||||
impl ResponsesApiCallAttempt {
|
||||
fn new(turn_responses_call_index: u64, input_items: &[ResponseItem]) -> Self {
|
||||
let input_item_count = input_items.len();
|
||||
let items = responses_api_input_items_metadata(input_items);
|
||||
Self {
|
||||
turn_responses_call_index,
|
||||
started_at: now_unix_seconds(),
|
||||
started_instant: Instant::now(),
|
||||
stream_started: false,
|
||||
input_item_count,
|
||||
output_item_count: 0,
|
||||
items,
|
||||
responses_id: None,
|
||||
token_usage: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn record_output_item(&mut self, item: &ResponseItem) {
|
||||
let item_index = self.output_item_count;
|
||||
self.items
|
||||
.push(responses_api_output_item_metadata(item_index, item));
|
||||
self.output_item_count += 1;
|
||||
}
|
||||
}
|
||||
|
||||
fn emit_responses_api_call_attempt(
|
||||
sess: &Session,
|
||||
turn_context: &TurnContext,
|
||||
attempt: ResponsesApiCallAttempt,
|
||||
status: CodexResponsesApiCallStatus,
|
||||
error: Option<String>,
|
||||
) {
|
||||
if !sess.enabled(Feature::GeneralAnalytics) {
|
||||
return;
|
||||
}
|
||||
if status == CodexResponsesApiCallStatus::Interrupted && !attempt.stream_started {
|
||||
return;
|
||||
}
|
||||
let input = CodexResponsesApiCallInput {
|
||||
responses_id: attempt.responses_id,
|
||||
turn_responses_call_index: attempt.turn_responses_call_index,
|
||||
status,
|
||||
error,
|
||||
started_at: attempt.started_at,
|
||||
completed_at: Some(now_unix_seconds()),
|
||||
duration_ms: Some(attempt.started_instant.elapsed().as_millis() as u64),
|
||||
input_item_count: attempt.input_item_count,
|
||||
output_item_count: attempt.output_item_count,
|
||||
items: attempt.items,
|
||||
token_usage: attempt.token_usage,
|
||||
};
|
||||
sess.services
|
||||
.analytics_events_client
|
||||
.track_responses_api_call(
|
||||
build_track_events_context(
|
||||
turn_context.model_info.slug.clone(),
|
||||
sess.conversation_id.to_string(),
|
||||
turn_context.sub_id.clone(),
|
||||
),
|
||||
input,
|
||||
);
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
#[instrument(level = "trace",
|
||||
skip_all,
|
||||
@@ -1047,6 +1133,7 @@ async fn run_sampling_request(
|
||||
turn_diff_tracker: SharedTurnDiffTracker,
|
||||
client_session: &mut ModelClientSession,
|
||||
turn_metadata_header: Option<&str>,
|
||||
next_turn_responses_call_index: &mut u64,
|
||||
input: Vec<ResponseItem>,
|
||||
explicitly_enabled_connectors: &HashSet<String>,
|
||||
skills_outcome: Option<&SkillLoadOutcome>,
|
||||
@@ -1097,6 +1184,10 @@ async fn run_sampling_request(
|
||||
turn_context.as_ref(),
|
||||
base_instructions.clone(),
|
||||
);
|
||||
let turn_responses_call_index = *next_turn_responses_call_index;
|
||||
*next_turn_responses_call_index = (*next_turn_responses_call_index).saturating_add(1);
|
||||
let mut responses_api_call_attempt =
|
||||
ResponsesApiCallAttempt::new(turn_responses_call_index, &prompt.input);
|
||||
let err = match try_run_sampling_request(
|
||||
tool_runtime.clone(),
|
||||
Arc::clone(&sess),
|
||||
@@ -1106,15 +1197,40 @@ async fn run_sampling_request(
|
||||
Arc::clone(&turn_diff_tracker),
|
||||
server_model_warning_emitted_for_turn,
|
||||
&prompt,
|
||||
&mut responses_api_call_attempt,
|
||||
cancellation_token.child_token(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(output) => {
|
||||
let status = if output.response_completed {
|
||||
CodexResponsesApiCallStatus::Completed
|
||||
} else {
|
||||
CodexResponsesApiCallStatus::Interrupted
|
||||
};
|
||||
let error = if output.response_completed {
|
||||
None
|
||||
} else {
|
||||
Some("stream preempted by pending input".to_string())
|
||||
};
|
||||
emit_responses_api_call_attempt(
|
||||
sess.as_ref(),
|
||||
turn_context.as_ref(),
|
||||
responses_api_call_attempt,
|
||||
status,
|
||||
error,
|
||||
);
|
||||
return Ok(output);
|
||||
}
|
||||
Err(CodexErr::ContextWindowExceeded) => {
|
||||
sess.set_total_tokens_full(&turn_context).await;
|
||||
emit_responses_api_call_attempt(
|
||||
sess.as_ref(),
|
||||
turn_context.as_ref(),
|
||||
responses_api_call_attempt,
|
||||
CodexResponsesApiCallStatus::Failed,
|
||||
Some("context window exceeded".to_string()),
|
||||
);
|
||||
return Err(CodexErr::ContextWindowExceeded);
|
||||
}
|
||||
Err(CodexErr::UsageLimitReached(e)) => {
|
||||
@@ -1122,12 +1238,32 @@ async fn run_sampling_request(
|
||||
if let Some(rate_limits) = rate_limits {
|
||||
sess.update_rate_limits(&turn_context, *rate_limits).await;
|
||||
}
|
||||
emit_responses_api_call_attempt(
|
||||
sess.as_ref(),
|
||||
turn_context.as_ref(),
|
||||
responses_api_call_attempt,
|
||||
CodexResponsesApiCallStatus::Failed,
|
||||
Some(e.to_string()),
|
||||
);
|
||||
return Err(CodexErr::UsageLimitReached(e));
|
||||
}
|
||||
Err(err) => err,
|
||||
};
|
||||
|
||||
if !err.is_retryable() {
|
||||
let status = if matches!(err, CodexErr::TurnAborted) {
|
||||
CodexResponsesApiCallStatus::Interrupted
|
||||
} else {
|
||||
CodexResponsesApiCallStatus::Failed
|
||||
};
|
||||
let error = Some(format!("{err:#}"));
|
||||
emit_responses_api_call_attempt(
|
||||
sess.as_ref(),
|
||||
turn_context.as_ref(),
|
||||
responses_api_call_attempt,
|
||||
status,
|
||||
error,
|
||||
);
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
@@ -1139,6 +1275,14 @@ async fn run_sampling_request(
|
||||
&turn_context.model_info,
|
||||
)
|
||||
{
|
||||
let error = Some(format!("{err:#}"));
|
||||
emit_responses_api_call_attempt(
|
||||
sess.as_ref(),
|
||||
turn_context.as_ref(),
|
||||
responses_api_call_attempt,
|
||||
CodexResponsesApiCallStatus::Failed,
|
||||
error,
|
||||
);
|
||||
sess.send_event(
|
||||
&turn_context,
|
||||
EventMsg::Warning(WarningEvent {
|
||||
@@ -1160,6 +1304,14 @@ async fn run_sampling_request(
|
||||
warn!(
|
||||
"stream disconnected - retrying sampling request ({retries}/{max_retries} in {delay:?})...",
|
||||
);
|
||||
let error = Some(format!("{err:#}"));
|
||||
emit_responses_api_call_attempt(
|
||||
sess.as_ref(),
|
||||
turn_context.as_ref(),
|
||||
responses_api_call_attempt,
|
||||
CodexResponsesApiCallStatus::Failed,
|
||||
error,
|
||||
);
|
||||
|
||||
// In release builds, hide the first websocket retry notification to reduce noisy
|
||||
// transient reconnect messages. In debug builds, keep full visibility for diagnosis.
|
||||
@@ -1179,6 +1331,14 @@ async fn run_sampling_request(
|
||||
}
|
||||
tokio::time::sleep(delay).await;
|
||||
} else {
|
||||
let error = Some(format!("{err:#}"));
|
||||
emit_responses_api_call_attempt(
|
||||
sess.as_ref(),
|
||||
turn_context.as_ref(),
|
||||
responses_api_call_attempt,
|
||||
CodexResponsesApiCallStatus::Failed,
|
||||
error,
|
||||
);
|
||||
return Err(err);
|
||||
}
|
||||
}
|
||||
@@ -1330,6 +1490,7 @@ pub(crate) async fn built_tools(
|
||||
struct SamplingRequestResult {
|
||||
needs_follow_up: bool,
|
||||
last_agent_message: Option<String>,
|
||||
response_completed: bool,
|
||||
}
|
||||
|
||||
/// Ephemeral per-response state for streaming a single proposed plan.
|
||||
@@ -1895,6 +2056,7 @@ async fn try_run_sampling_request(
|
||||
turn_diff_tracker: SharedTurnDiffTracker,
|
||||
server_model_warning_emitted_for_turn: &mut bool,
|
||||
prompt: &Prompt,
|
||||
responses_api_call_attempt: &mut ResponsesApiCallAttempt,
|
||||
cancellation_token: CancellationToken,
|
||||
) -> CodexResult<SamplingRequestResult> {
|
||||
feedback_tags!(
|
||||
@@ -1918,6 +2080,7 @@ async fn try_run_sampling_request(
|
||||
.instrument(trace_span!("stream_request"))
|
||||
.or_cancel(&cancellation_token)
|
||||
.await??;
|
||||
responses_api_call_attempt.stream_started = true;
|
||||
let mut in_flight: FuturesOrdered<BoxFuture<'static, CodexResult<ResponseInputItem>>> =
|
||||
FuturesOrdered::new();
|
||||
let mut needs_follow_up = false;
|
||||
@@ -1970,6 +2133,7 @@ async fn try_run_sampling_request(
|
||||
match event {
|
||||
ResponseEvent::Created => {}
|
||||
ResponseEvent::OutputItemDone(item) => {
|
||||
responses_api_call_attempt.record_output_item(&item);
|
||||
if let Some((_, mut consumer)) = active_tool_argument_diff_consumer.take()
|
||||
&& let Some(event) = consumer.flush_on_complete()
|
||||
{
|
||||
@@ -2049,6 +2213,7 @@ async fn try_run_sampling_request(
|
||||
break Ok(SamplingRequestResult {
|
||||
needs_follow_up: true,
|
||||
last_agent_message,
|
||||
response_completed: false,
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -2140,9 +2305,11 @@ async fn try_run_sampling_request(
|
||||
sess.services.models_manager.refresh_if_new_etag(etag).await;
|
||||
}
|
||||
ResponseEvent::Completed {
|
||||
response_id: _,
|
||||
response_id,
|
||||
token_usage,
|
||||
} => {
|
||||
responses_api_call_attempt.responses_id = Some(response_id);
|
||||
responses_api_call_attempt.token_usage = token_usage.clone();
|
||||
flush_assistant_text_segments_all(
|
||||
&sess,
|
||||
&turn_context,
|
||||
@@ -2157,6 +2324,7 @@ async fn try_run_sampling_request(
|
||||
break Ok(SamplingRequestResult {
|
||||
needs_follow_up,
|
||||
last_agent_message,
|
||||
response_completed: true,
|
||||
});
|
||||
}
|
||||
ResponseEvent::OutputTextDelta(delta) => {
|
||||
|
||||
Reference in New Issue
Block a user