clone/codex
1
0
Fork 0
mirror of https://github.com/openai/codex.git synced 2026-04-27 08:05:51 +00:00
Code Issues Packages Projects Releases Wiki Activity

[codex-analytics] feature plumbing and emittance

Browse Source
This commit is contained in:
rhan-oai
2026-04-06 14:46:08 -07:00
parent 964a9a5aed
commit f3ff02108d
21 changed files with 1274 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

398
codex-rs/analytics/src/analytics_client_tests.rs
View File

@@ -6,6 +6,7 @@ use crate::events::CodexAppUsedEventRequest;
use crate::events::CodexPluginEventRequest;
use crate::events::CodexPluginUsedEventRequest;
use crate::events::CodexRuntimeMetadata;
use crate::events::CodexTurnEventRequest;
use crate::events::ThreadInitializationMode;
use crate::events::ThreadInitializedEvent;
use crate::events::ThreadInitializedEventParams;
@@ -27,28 +28,43 @@ use crate::facts::SkillInvocation;
use crate::facts::SkillInvokedInput;
use crate::facts::SubAgentThreadStartedInput;
use crate::facts::TrackEventsContext;
use crate::facts::TurnResolvedConfigFact;
use crate::facts::TurnStatus;
use crate::reducer::AnalyticsReducer;
use crate::reducer::normalize_path_for_skill_id;
use crate::reducer::skill_id_for_local_skill;
use codex_app_server_protocol::ApprovalsReviewer as AppServerApprovalsReviewer;
use codex_app_server_protocol::AskForApproval as AppServerAskForApproval;
use codex_app_server_protocol::ClientInfo;
use codex_app_server_protocol::ClientRequest;
use codex_app_server_protocol::ClientResponse;
use codex_app_server_protocol::InitializeCapabilities;
use codex_app_server_protocol::InitializeParams;
use codex_app_server_protocol::RequestId;
use codex_app_server_protocol::SandboxPolicy as AppServerSandboxPolicy;
use codex_app_server_protocol::ServerNotification;
use codex_app_server_protocol::SessionSource as AppServerSessionSource;
use codex_app_server_protocol::Thread;
use codex_app_server_protocol::ThreadResumeResponse;
use codex_app_server_protocol::ThreadStartResponse;
use codex_app_server_protocol::ThreadStatus as AppServerThreadStatus;
use codex_app_server_protocol::Turn;
use codex_app_server_protocol::TurnCompletedNotification;
use codex_app_server_protocol::TurnError as AppServerTurnError;
use codex_app_server_protocol::TurnStartParams;
use codex_app_server_protocol::TurnStartedNotification;
use codex_app_server_protocol::TurnStatus as AppServerTurnStatus;
use codex_app_server_protocol::UserInput;
use codex_login::default_client::DEFAULT_ORIGINATOR;
use codex_login::default_client::originator;
use codex_plugin::AppConnectorId;
use codex_plugin::PluginCapabilitySummary;
use codex_plugin::PluginId;
use codex_plugin::PluginTelemetryMetadata;
use codex_protocol::config_types::ApprovalsReviewer;
use codex_protocol::config_types::ModeKind;
use codex_protocol::protocol::AskForApproval;
use codex_protocol::protocol::SandboxPolicy;
use codex_protocol::protocol::SubAgentSource;
use pretty_assertions::assert_eq;
use serde_json::json;
@@ -114,6 +130,179 @@ fn sample_thread_resume_response(thread_id: &str, ephemeral: bool, model: &str)
}
}
fn sample_turn_start_request(thread_id: &str, request_id: i64) -> ClientRequest {
ClientRequest::TurnStart {
request_id: RequestId::Integer(request_id),
params: TurnStartParams {
thread_id: thread_id.to_string(),
input: vec![
UserInput::Text {
text: "hello".to_string(),
text_elements: vec![],
},
UserInput::Image {
url: "https://example.com/a.png".to_string(),
},
],
..Default::default()
},
}
}
fn sample_turn_start_response(turn_id: &str, request_id: i64) -> ClientResponse {
ClientResponse::TurnStart {
request_id: RequestId::Integer(request_id),
response: codex_app_server_protocol::TurnStartResponse {
turn: Turn {
id: turn_id.to_string(),
items: vec![],
status: AppServerTurnStatus::InProgress,
error: None,
started_at: None,
completed_at: None,
duration_ms: None,
},
},
}
}
fn sample_turn_started_notification(thread_id: &str, turn_id: &str) -> ServerNotification {
ServerNotification::TurnStarted(TurnStartedNotification {
thread_id: thread_id.to_string(),
turn: Turn {
id: turn_id.to_string(),
items: vec![],
status: AppServerTurnStatus::InProgress,
error: None,
started_at: Some(455),
completed_at: None,
duration_ms: None,
},
})
}
fn sample_turn_completed_notification(
thread_id: &str,
turn_id: &str,
status: AppServerTurnStatus,
codex_error_info: Option<codex_app_server_protocol::CodexErrorInfo>,
) -> ServerNotification {
ServerNotification::TurnCompleted(TurnCompletedNotification {
thread_id: thread_id.to_string(),
turn: Turn {
id: turn_id.to_string(),
items: vec![],
status,
error: codex_error_info.map(|codex_error_info| AppServerTurnError {
message: "turn failed".to_string(),
codex_error_info: Some(codex_error_info),
additional_details: None,
}),
started_at: None,
completed_at: Some(456),
duration_ms: Some(1234),
},
})
}
fn sample_turn_resolved_config(turn_id: &str) -> TurnResolvedConfigFact {
TurnResolvedConfigFact {
turn_id: turn_id.to_string(),
thread_id: "thread-2".to_string(),
num_input_images: 1,
submission_type: None,
model: "gpt-5".to_string(),
model_provider: "openai".to_string(),
sandbox_policy: SandboxPolicy::new_read_only_policy(),
reasoning_effort: None,
reasoning_summary: None,
service_tier: None,
approval_policy: AskForApproval::OnRequest,
approvals_reviewer: ApprovalsReviewer::GuardianSubagent,
sandbox_network_access: true,
collaboration_mode: ModeKind::Plan,
personality: None,
is_first_turn: true,
}
}
async fn ingest_turn_prerequisites(
reducer: &mut AnalyticsReducer,
out: &mut Vec<TrackEventRequest>,
include_initialize: bool,
include_resolved_config: bool,
include_started: bool,
) {
if include_initialize {
reducer
.ingest(
AnalyticsFact::Initialize {
connection_id: 7,
params: InitializeParams {
client_info: ClientInfo {
name: "codex-tui".to_string(),
title: None,
version: "1.0.0".to_string(),
},
capabilities: None,
},
product_client_id: "codex-tui".to_string(),
runtime: CodexRuntimeMetadata {
codex_rs_version: "0.1.0".to_string(),
runtime_os: "macos".to_string(),
runtime_os_version: "15.3.1".to_string(),
runtime_arch: "aarch64".to_string(),
},
rpc_transport: AppServerRpcTransport::Stdio,
},
out,
)
.await;
}
reducer
.ingest(
AnalyticsFact::Request {
connection_id: 7,
request_id: RequestId::Integer(3),
request: Box::new(sample_turn_start_request("thread-2", /*request_id*/ 3)),
},
out,
)
.await;
reducer
.ingest(
AnalyticsFact::Response {
connection_id: 7,
response: Box::new(sample_turn_start_response("turn-2", /*request_id*/ 3)),
},
out,
)
.await;
if include_resolved_config {
reducer
.ingest(
AnalyticsFact::Custom(CustomAnalyticsFact::TurnResolvedConfig(Box::new(
sample_turn_resolved_config("turn-2"),
))),
out,
)
.await;
}
if include_started {
reducer
.ingest(
AnalyticsFact::Notification(Box::new(sample_turn_started_notification(
"thread-2", "turn-2",
))),
out,
)
.await;
}
}
fn expected_absolute_path(path: &PathBuf) -> String {
std::fs::canonicalize(path)
.unwrap_or_else(|_| path.to_path_buf())
@@ -823,6 +1012,215 @@ async fn reducer_ingests_plugin_state_changed_fact() {
);
}
#[test]
fn turn_event_serializes_expected_shape() {
let event = TrackEventRequest::TurnEvent(Box::new(CodexTurnEventRequest {
event_type: "codex_turn_event",
event_params: crate::events::CodexTurnEventParams {
thread_id: "thread-2".to_string(),
turn_id: "turn-2".to_string(),
product_client_id: "codex-tui".to_string(),
submission_type: None,
model: Some("gpt-5".to_string()),
model_provider: "openai".to_string(),
sandbox_policy: Some("read_only"),
reasoning_effort: Some("high".to_string()),
reasoning_summary: Some("detailed".to_string()),
service_tier: "flex".to_string(),
approval_policy: "on-request".to_string(),
approvals_reviewer: "guardian_subagent".to_string(),
sandbox_network_access: true,
collaboration_mode: Some("plan"),
personality: Some("pragmatic".to_string()),
num_input_images: 2,
is_first_turn: true,
status: Some(TurnStatus::Completed),
turn_error: None,
steer_count: None,
total_tool_call_count: None,
shell_command_count: None,
file_change_count: None,
mcp_tool_call_count: None,
dynamic_tool_call_count: None,
subagent_tool_call_count: None,
web_search_count: None,
image_generation_count: None,
duration_ms: Some(1234),
started_at: Some(455),
completed_at: Some(456),
},
}));
let payload = serde_json::to_value(&event).expect("serialize turn event");
assert_eq!(
payload,
json!({
"event_type": "codex_turn_event",
"event_params": {
"thread_id": "thread-2",
"turn_id": "turn-2",
"product_client_id": "codex-tui",
"submission_type": null,
"model": "gpt-5",
"model_provider": "openai",
"sandbox_policy": "read_only",
"reasoning_effort": "high",
"reasoning_summary": "detailed",
"service_tier": "flex",
"approval_policy": "on-request",
"approvals_reviewer": "guardian_subagent",
"sandbox_network_access": true,
"collaboration_mode": "plan",
"personality": "pragmatic",
"num_input_images": 2,
"is_first_turn": true,
"status": "completed",
"turn_error": null,
"steer_count": null,
"total_tool_call_count": null,
"shell_command_count": null,
"file_change_count": null,
"mcp_tool_call_count": null,
"dynamic_tool_call_count": null,
"subagent_tool_call_count": null,
"web_search_count": null,
"image_generation_count": null,
"duration_ms": 1234,
"created_at": 455,
"completed_at": 456
}
})
);
}
#[tokio::test]
async fn turn_lifecycle_emits_turn_event() {
let mut reducer = AnalyticsReducer::default();
let mut out = Vec::new();
ingest_turn_prerequisites(
&mut reducer,
&mut out,
/*include_initialize*/ true,
/*include_resolved_config*/ true,
/*include_started*/ true,
)
.await;
reducer
.ingest(
AnalyticsFact::Notification(Box::new(sample_turn_completed_notification(
"thread-2",
"turn-2",
AppServerTurnStatus::Completed,
/*codex_error_info*/ None,
))),
&mut out,
)
.await;
assert_eq!(out.len(), 1);
let payload = serde_json::to_value(&out[0]).expect("serialize turn event");
assert_eq!(payload["event_type"], json!("codex_turn_event"));
assert_eq!(payload["event_params"]["thread_id"], json!("thread-2"));
assert_eq!(payload["event_params"]["turn_id"], json!("turn-2"));
assert_eq!(
payload["event_params"]["product_client_id"],
json!("codex-tui")
);
assert_eq!(payload["event_params"]["num_input_images"], json!(1));
assert_eq!(payload["event_params"]["status"], json!("completed"));
assert_eq!(payload["event_params"]["started_at"], json!(455));
assert_eq!(payload["event_params"]["completed_at"], json!(456));
assert_eq!(payload["event_params"]["duration_ms"], json!(1234));
}
#[tokio::test]
async fn turn_does_not_emit_without_required_prerequisites() {
let mut reducer = AnalyticsReducer::default();
let mut out = Vec::new();
ingest_turn_prerequisites(
&mut reducer,
&mut out,
/*include_initialize*/ false,
/*include_resolved_config*/ true,
/*include_started*/ false,
)
.await;
reducer
.ingest(
AnalyticsFact::Notification(Box::new(sample_turn_completed_notification(
"thread-2",
"turn-2",
AppServerTurnStatus::Completed,
/*codex_error_info*/ None,
))),
&mut out,
)
.await;
assert_eq!(out.len(), 1);
let payload = serde_json::to_value(&out[0]).expect("serialize turn event");
assert_eq!(
payload["event_params"]["product_client_id"],
json!(originator().value)
);
let mut reducer = AnalyticsReducer::default();
let mut out = Vec::new();
ingest_turn_prerequisites(
&mut reducer,
&mut out,
/*include_initialize*/ true,
/*include_resolved_config*/ false,
/*include_started*/ false,
)
.await;
reducer
.ingest(
AnalyticsFact::Notification(Box::new(sample_turn_completed_notification(
"thread-2",
"turn-2",
AppServerTurnStatus::Completed,
/*codex_error_info*/ None,
))),
&mut out,
)
.await;
assert!(out.is_empty());
}
#[tokio::test]
async fn turn_completed_without_started_notification_emits_null_started_at() {
let mut reducer = AnalyticsReducer::default();
let mut out = Vec::new();
ingest_turn_prerequisites(
&mut reducer,
&mut out,
/*include_initialize*/ true,
/*include_resolved_config*/ true,
/*include_started*/ false,
)
.await;
reducer
.ingest(
AnalyticsFact::Notification(Box::new(sample_turn_completed_notification(
"thread-2",
"turn-2",
AppServerTurnStatus::Completed,
/*codex_error_info*/ None,
))),
&mut out,
)
.await;
let payload = serde_json::to_value(&out[0]).expect("serialize turn event");
assert_eq!(payload["event_params"]["started_at"], json!(null));
assert_eq!(payload["event_params"]["duration_ms"], json!(1234));
}
fn sample_plugin_metadata() -> PluginTelemetryMetadata {
PluginTelemetryMetadata {
plugin_id: PluginId::parse("sample@test").expect("valid plugin id"),