mirror of
https://github.com/openai/codex.git
synced 2026-04-23 22:24:57 +00:00
analytics: reduce responses api call facts
This commit is contained in:
rhan-oai
@@ -6,6 +6,8 @@ use crate::events::CodexAppUsedEventRequest;
|
||||
use crate::events::CodexCompactionEventRequest;
|
||||
use crate::events::CodexPluginEventRequest;
|
||||
use crate::events::CodexPluginUsedEventRequest;
|
||||
use crate::events::CodexResponsesApiCallEventParams;
|
||||
use crate::events::CodexResponsesApiCallEventRequest;
|
||||
use crate::events::CodexRuntimeMetadata;
|
||||
use crate::events::CodexTurnEventRequest;
|
||||
use crate::events::ThreadInitializedEvent;
|
||||
@@ -45,6 +47,13 @@ use crate::facts::TurnTokenUsageFact;
|
||||
use crate::reducer::AnalyticsReducer;
|
||||
use crate::reducer::normalize_path_for_skill_id;
|
||||
use crate::reducer::skill_id_for_local_skill;
|
||||
use crate::responses_api::CodexResponseItemRole;
|
||||
use crate::responses_api::CodexResponseItemType;
|
||||
use crate::responses_api::CodexResponseMessagePhase;
|
||||
use crate::responses_api::CodexResponsesApiCallFact;
|
||||
use crate::responses_api::CodexResponsesApiCallStatus;
|
||||
use crate::responses_api::CodexResponsesApiItemMetadata;
|
||||
use crate::responses_api::CodexResponsesApiItemPhase;
|
||||
use codex_app_server_protocol::ApprovalsReviewer as AppServerApprovalsReviewer;
|
||||
use codex_app_server_protocol::AskForApproval as AppServerAskForApproval;
|
||||
use codex_app_server_protocol::ClientInfo;
|
||||
@@ -259,6 +268,30 @@ fn sample_turn_token_usage_fact(thread_id: &str, turn_id: &str) -> TurnTokenUsag
|
||||
}
|
||||
}
|
||||
|
||||
fn sample_responses_api_item_metadata(
|
||||
item_phase: CodexResponsesApiItemPhase,
|
||||
item_index: usize,
|
||||
) -> CodexResponsesApiItemMetadata {
|
||||
CodexResponsesApiItemMetadata {
|
||||
item_phase,
|
||||
item_index,
|
||||
response_item_type: CodexResponseItemType::Message,
|
||||
role: Some(CodexResponseItemRole::Assistant),
|
||||
status: Some("completed".to_string()),
|
||||
message_phase: Some(CodexResponseMessagePhase::FinalAnswer),
|
||||
call_id: None,
|
||||
tool_name: None,
|
||||
content_bytes: Some(12),
|
||||
arguments_bytes: None,
|
||||
output_bytes: None,
|
||||
encrypted_content_bytes: None,
|
||||
image_generation_revised_prompt_bytes: None,
|
||||
image_generation_result_bytes: None,
|
||||
text_part_count: Some(1),
|
||||
image_part_count: Some(0),
|
||||
}
|
||||
}
|
||||
|
||||
fn sample_turn_completed_notification(
|
||||
thread_id: &str,
|
||||
turn_id: &str,
|
||||
@@ -738,6 +771,93 @@ fn compaction_event_serializes_expected_shape() {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn responses_api_call_event_serializes_expected_shape() {
|
||||
let event = TrackEventRequest::ResponsesApiCall(Box::new(CodexResponsesApiCallEventRequest {
|
||||
event_type: "codex_responses_api_call_event",
|
||||
event_params: CodexResponsesApiCallEventParams {
|
||||
thread_id: "thread-1".to_string(),
|
||||
turn_id: "turn-1".to_string(),
|
||||
ephemeral: false,
|
||||
thread_source: Some("user".to_string()),
|
||||
initialization_mode: ThreadInitializationMode::New,
|
||||
subagent_source: None,
|
||||
parent_thread_id: None,
|
||||
app_server_client: sample_app_server_client_metadata(),
|
||||
runtime: sample_runtime_metadata(),
|
||||
responses_id: Some("resp_123".to_string()),
|
||||
turn_responses_call_index: 2,
|
||||
model: Some("gpt-5".to_string()),
|
||||
model_provider: Some("openai".to_string()),
|
||||
reasoning_effort: Some("high".to_string()),
|
||||
status: CodexResponsesApiCallStatus::Completed,
|
||||
error: None,
|
||||
started_at: 100,
|
||||
completed_at: Some(102),
|
||||
duration_ms: Some(2345),
|
||||
input_item_count: 1,
|
||||
output_item_count: 1,
|
||||
input_tokens: Some(10),
|
||||
cached_input_tokens: Some(4),
|
||||
output_tokens: Some(20),
|
||||
reasoning_output_tokens: Some(3),
|
||||
total_tokens: Some(30),
|
||||
items: vec![
|
||||
sample_responses_api_item_metadata(CodexResponsesApiItemPhase::Input, 0),
|
||||
sample_responses_api_item_metadata(CodexResponsesApiItemPhase::Output, 0),
|
||||
],
|
||||
},
|
||||
}));
|
||||
|
||||
let payload = serde_json::to_value(&event).expect("serialize responses api call event");
|
||||
|
||||
assert_eq!(payload["event_type"], "codex_responses_api_call_event");
|
||||
assert_eq!(payload["event_params"]["thread_id"], "thread-1");
|
||||
assert_eq!(payload["event_params"]["turn_id"], "turn-1");
|
||||
assert_eq!(payload["event_params"]["ephemeral"], false);
|
||||
assert_eq!(payload["event_params"]["thread_source"], "user");
|
||||
assert_eq!(payload["event_params"]["initialization_mode"], "new");
|
||||
assert_eq!(
|
||||
payload["event_params"]["app_server_client"]["product_client_id"],
|
||||
DEFAULT_ORIGINATOR
|
||||
);
|
||||
assert_eq!(
|
||||
payload["event_params"]["app_server_client"]["rpc_transport"],
|
||||
"stdio"
|
||||
);
|
||||
assert_eq!(
|
||||
payload["event_params"]["runtime"]["codex_rs_version"],
|
||||
"0.1.0"
|
||||
);
|
||||
assert_eq!(payload["event_params"]["responses_id"], "resp_123");
|
||||
assert_eq!(payload["event_params"]["turn_responses_call_index"], 2);
|
||||
assert_eq!(payload["event_params"]["model"], "gpt-5");
|
||||
assert_eq!(payload["event_params"]["model_provider"], "openai");
|
||||
assert_eq!(payload["event_params"]["reasoning_effort"], "high");
|
||||
assert_eq!(payload["event_params"]["status"], "completed");
|
||||
assert_eq!(payload["event_params"]["error"], json!(null));
|
||||
assert_eq!(payload["event_params"]["started_at"], 100);
|
||||
assert_eq!(payload["event_params"]["completed_at"], 102);
|
||||
assert_eq!(payload["event_params"]["duration_ms"], 2345);
|
||||
assert_eq!(payload["event_params"]["input_item_count"], 1);
|
||||
assert_eq!(payload["event_params"]["output_item_count"], 1);
|
||||
assert_eq!(payload["event_params"]["input_tokens"], 10);
|
||||
assert_eq!(payload["event_params"]["cached_input_tokens"], 4);
|
||||
assert_eq!(payload["event_params"]["output_tokens"], 20);
|
||||
assert_eq!(payload["event_params"]["reasoning_output_tokens"], 3);
|
||||
assert_eq!(payload["event_params"]["total_tokens"], 30);
|
||||
assert_eq!(payload["event_params"]["items"][0]["item_phase"], "input");
|
||||
assert_eq!(payload["event_params"]["items"][0]["response_item_type"], "message");
|
||||
assert_eq!(payload["event_params"]["items"][0]["role"], "assistant");
|
||||
assert_eq!(
|
||||
payload["event_params"]["items"][0]["message_phase"],
|
||||
"final_answer"
|
||||
);
|
||||
assert_eq!(payload["event_params"]["items"][0]["text_part_count"], 1);
|
||||
assert_eq!(payload["event_params"]["items"][0]["image_part_count"], 0);
|
||||
assert_eq!(payload["event_params"]["items"][1]["item_phase"], "output");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn app_used_dedupe_is_keyed_by_turn_and_connector() {
|
||||
let (sender, _receiver) = mpsc::channel(1);
|
||||
@@ -1043,6 +1163,82 @@ async fn compaction_event_ingests_custom_fact() {
|
||||
assert_eq!(payload[0]["event_params"]["status"], "failed");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn responses_api_call_event_ingests_custom_fact() {
|
||||
let mut reducer = AnalyticsReducer::default();
|
||||
let mut events = Vec::new();
|
||||
|
||||
ingest_turn_prerequisites(
|
||||
&mut reducer,
|
||||
&mut events,
|
||||
/*include_initialize*/ true,
|
||||
/*include_resolved_config*/ true,
|
||||
/*include_started*/ false,
|
||||
/*include_token_usage*/ false,
|
||||
)
|
||||
.await;
|
||||
events.clear();
|
||||
|
||||
reducer
|
||||
.ingest(
|
||||
AnalyticsFact::Custom(CustomAnalyticsFact::ResponsesApiCall(Box::new(
|
||||
CodexResponsesApiCallFact {
|
||||
thread_id: "thread-2".to_string(),
|
||||
turn_id: "turn-2".to_string(),
|
||||
responses_id: Some("resp_456".to_string()),
|
||||
turn_responses_call_index: 1,
|
||||
status: CodexResponsesApiCallStatus::Completed,
|
||||
error: None,
|
||||
started_at: 100,
|
||||
completed_at: Some(101),
|
||||
duration_ms: Some(1200),
|
||||
input_item_count: 1,
|
||||
output_item_count: 1,
|
||||
input_tokens: Some(123),
|
||||
cached_input_tokens: Some(45),
|
||||
output_tokens: Some(140),
|
||||
reasoning_output_tokens: Some(13),
|
||||
total_tokens: Some(321),
|
||||
items: vec![
|
||||
sample_responses_api_item_metadata(CodexResponsesApiItemPhase::Input, 0),
|
||||
sample_responses_api_item_metadata(CodexResponsesApiItemPhase::Output, 0),
|
||||
],
|
||||
},
|
||||
))),
|
||||
&mut events,
|
||||
)
|
||||
.await;
|
||||
|
||||
let payload = serde_json::to_value(&events).expect("serialize events");
|
||||
assert_eq!(payload.as_array().expect("events array").len(), 1);
|
||||
assert_eq!(payload[0]["event_type"], "codex_responses_api_call_event");
|
||||
assert_eq!(payload[0]["event_params"]["thread_id"], "thread-2");
|
||||
assert_eq!(payload[0]["event_params"]["turn_id"], "turn-2");
|
||||
assert_eq!(payload[0]["event_params"]["ephemeral"], false);
|
||||
assert_eq!(payload[0]["event_params"]["thread_source"], "user");
|
||||
assert_eq!(payload[0]["event_params"]["initialization_mode"], "new");
|
||||
assert_eq!(
|
||||
payload[0]["event_params"]["app_server_client"]["product_client_id"],
|
||||
"codex-tui"
|
||||
);
|
||||
assert_eq!(payload[0]["event_params"]["runtime"]["runtime_os"], "macos");
|
||||
assert_eq!(payload[0]["event_params"]["responses_id"], "resp_456");
|
||||
assert_eq!(payload[0]["event_params"]["turn_responses_call_index"], 1);
|
||||
assert_eq!(payload[0]["event_params"]["model"], "gpt-5");
|
||||
assert_eq!(payload[0]["event_params"]["model_provider"], "openai");
|
||||
assert_eq!(payload[0]["event_params"]["reasoning_effort"], json!(null));
|
||||
assert_eq!(payload[0]["event_params"]["status"], "completed");
|
||||
assert_eq!(payload[0]["event_params"]["input_tokens"], 123);
|
||||
assert_eq!(
|
||||
payload[0]["event_params"]["items"][0]["item_phase"],
|
||||
"input"
|
||||
);
|
||||
assert_eq!(
|
||||
payload[0]["event_params"]["items"][1]["item_phase"],
|
||||
"output"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn subagent_thread_started_review_serializes_expected_shape() {
|
||||
let event = TrackEventRequest::ThreadInitialized(subagent_thread_started_event_request(
|
||||
|
||||
@@ -18,6 +18,7 @@ use crate::facts::TrackEventsContext;
|
||||
use crate::facts::TurnResolvedConfigFact;
|
||||
use crate::facts::TurnTokenUsageFact;
|
||||
use crate::reducer::AnalyticsReducer;
|
||||
use crate::responses_api::CodexResponsesApiCallFact;
|
||||
use codex_app_server_protocol::ClientRequest;
|
||||
use codex_app_server_protocol::ClientResponse;
|
||||
use codex_app_server_protocol::InitializeParams;
|
||||
@@ -206,6 +207,12 @@ impl AnalyticsEventsClient {
|
||||
)));
|
||||
}
|
||||
|
||||
pub fn track_responses_api_call(&self, event: CodexResponsesApiCallFact) {
|
||||
self.record_fact(AnalyticsFact::Custom(
|
||||
CustomAnalyticsFact::ResponsesApiCall(Box::new(event)),
|
||||
));
|
||||
}
|
||||
|
||||
pub fn track_turn_resolved_config(&self, fact: TurnResolvedConfigFact) {
|
||||
self.record_fact(AnalyticsFact::Custom(
|
||||
CustomAnalyticsFact::TurnResolvedConfig(Box::new(fact)),
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
use crate::events::AppServerRpcTransport;
|
||||
use crate::events::CodexRuntimeMetadata;
|
||||
use crate::events::GuardianReviewEventParams;
|
||||
use crate::responses_api::CodexResponsesApiCallFact;
|
||||
use codex_app_server_protocol::ClientRequest;
|
||||
use codex_app_server_protocol::ClientResponse;
|
||||
use codex_app_server_protocol::InitializeParams;
|
||||
@@ -292,6 +293,7 @@ pub(crate) enum AnalyticsFact {
|
||||
pub(crate) enum CustomAnalyticsFact {
|
||||
SubAgentThreadStarted(SubAgentThreadStartedInput),
|
||||
Compaction(Box<CodexCompactionEvent>),
|
||||
ResponsesApiCall(Box<CodexResponsesApiCallFact>),
|
||||
GuardianReview(Box<GuardianReviewEventParams>),
|
||||
TurnResolvedConfig(Box<TurnResolvedConfigFact>),
|
||||
TurnTokenUsage(Box<TurnTokenUsageFact>),
|
||||
|
||||
@@ -5,6 +5,8 @@ use crate::events::CodexAppUsedEventRequest;
|
||||
use crate::events::CodexCompactionEventRequest;
|
||||
use crate::events::CodexPluginEventRequest;
|
||||
use crate::events::CodexPluginUsedEventRequest;
|
||||
use crate::events::CodexResponsesApiCallEventParams;
|
||||
use crate::events::CodexResponsesApiCallEventRequest;
|
||||
use crate::events::CodexRuntimeMetadata;
|
||||
use crate::events::CodexTurnEventParams;
|
||||
use crate::events::CodexTurnEventRequest;
|
||||
@@ -44,6 +46,7 @@ use crate::facts::TurnSteerRejectionReason;
|
||||
use crate::facts::TurnSteerResult;
|
||||
use crate::facts::TurnTokenUsageFact;
|
||||
use crate::now_unix_seconds;
|
||||
use crate::responses_api::CodexResponsesApiCallFact;
|
||||
use codex_app_server_protocol::ClientRequest;
|
||||
use codex_app_server_protocol::ClientResponse;
|
||||
use codex_app_server_protocol::CodexErrorInfo;
|
||||
@@ -199,6 +202,9 @@ impl AnalyticsReducer {
|
||||
CustomAnalyticsFact::Compaction(input) => {
|
||||
self.ingest_compaction(*input, out);
|
||||
}
|
||||
CustomAnalyticsFact::ResponsesApiCall(input) => {
|
||||
self.ingest_responses_api_call(*input, out);
|
||||
}
|
||||
CustomAnalyticsFact::GuardianReview(input) => {
|
||||
self.ingest_guardian_review(*input, out);
|
||||
}
|
||||
@@ -725,6 +731,91 @@ impl AnalyticsReducer {
|
||||
)));
|
||||
}
|
||||
|
||||
fn ingest_responses_api_call(
|
||||
&mut self,
|
||||
input: CodexResponsesApiCallFact,
|
||||
out: &mut Vec<TrackEventRequest>,
|
||||
) {
|
||||
let Some(connection_id) = self.thread_connections.get(&input.thread_id) else {
|
||||
tracing::warn!(
|
||||
thread_id = %input.thread_id,
|
||||
turn_id = %input.turn_id,
|
||||
"dropping responses api call analytics event: missing thread connection metadata"
|
||||
);
|
||||
return;
|
||||
};
|
||||
let Some(connection_state) = self.connections.get(connection_id) else {
|
||||
tracing::warn!(
|
||||
thread_id = %input.thread_id,
|
||||
turn_id = %input.turn_id,
|
||||
connection_id,
|
||||
"dropping responses api call analytics event: missing connection metadata"
|
||||
);
|
||||
return;
|
||||
};
|
||||
let Some(thread_metadata) = self.thread_metadata.get(&input.thread_id) else {
|
||||
tracing::warn!(
|
||||
thread_id = %input.thread_id,
|
||||
turn_id = %input.turn_id,
|
||||
"dropping responses api call analytics event: missing thread lifecycle metadata"
|
||||
);
|
||||
return;
|
||||
};
|
||||
let Some(turn_state) = self.turns.get(&input.turn_id) else {
|
||||
tracing::warn!(
|
||||
thread_id = %input.thread_id,
|
||||
turn_id = %input.turn_id,
|
||||
"dropping responses api call analytics event: missing turn metadata"
|
||||
);
|
||||
return;
|
||||
};
|
||||
let Some(resolved_config) = turn_state.resolved_config.as_ref() else {
|
||||
tracing::warn!(
|
||||
thread_id = %input.thread_id,
|
||||
turn_id = %input.turn_id,
|
||||
"dropping responses api call analytics event: missing turn resolved config"
|
||||
);
|
||||
return;
|
||||
};
|
||||
|
||||
out.push(TrackEventRequest::ResponsesApiCall(Box::new(
|
||||
CodexResponsesApiCallEventRequest {
|
||||
event_type: "codex_responses_api_call_event",
|
||||
event_params: CodexResponsesApiCallEventParams {
|
||||
thread_id: input.thread_id,
|
||||
turn_id: input.turn_id,
|
||||
ephemeral: resolved_config.ephemeral,
|
||||
thread_source: thread_metadata.thread_source.map(str::to_string),
|
||||
initialization_mode: thread_metadata.initialization_mode,
|
||||
subagent_source: thread_metadata.subagent_source.clone(),
|
||||
parent_thread_id: thread_metadata.parent_thread_id.clone(),
|
||||
app_server_client: connection_state.app_server_client.clone(),
|
||||
runtime: connection_state.runtime.clone(),
|
||||
responses_id: input.responses_id,
|
||||
turn_responses_call_index: input.turn_responses_call_index,
|
||||
model: Some(resolved_config.model.clone()),
|
||||
model_provider: Some(resolved_config.model_provider.clone()),
|
||||
reasoning_effort: resolved_config
|
||||
.reasoning_effort
|
||||
.map(|value| value.to_string()),
|
||||
status: input.status,
|
||||
error: input.error,
|
||||
started_at: input.started_at,
|
||||
completed_at: input.completed_at,
|
||||
duration_ms: input.duration_ms,
|
||||
input_item_count: input.input_item_count,
|
||||
output_item_count: input.output_item_count,
|
||||
input_tokens: input.input_tokens,
|
||||
cached_input_tokens: input.cached_input_tokens,
|
||||
output_tokens: input.output_tokens,
|
||||
reasoning_output_tokens: input.reasoning_output_tokens,
|
||||
total_tokens: input.total_tokens,
|
||||
items: input.items,
|
||||
},
|
||||
},
|
||||
)));
|
||||
}
|
||||
|
||||
fn ingest_turn_steer_response(
|
||||
&mut self,
|
||||
connection_id: u64,
|
||||
|
||||
Reference in New Issue
Block a user