diff --git a/codex-rs/analytics/Cargo.toml b/codex-rs/analytics/Cargo.toml index 2239c4bc36..0f36373145 100644 --- a/codex-rs/analytics/Cargo.toml +++ b/codex-rs/analytics/Cargo.toml @@ -20,6 +20,7 @@ codex-plugin = { workspace = true } codex-protocol = { workspace = true } os_info = { workspace = true } serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true } sha1 = { workspace = true } tokio = { workspace = true, features = [ "macros", @@ -29,4 +30,3 @@ tracing = { workspace = true, features = ["log"] } [dev-dependencies] pretty_assertions = { workspace = true } -serde_json = { workspace = true } diff --git a/codex-rs/analytics/src/analytics_client_tests.rs b/codex-rs/analytics/src/analytics_client_tests.rs index 73ea42d760..9aaf0e3cc6 100644 --- a/codex-rs/analytics/src/analytics_client_tests.rs +++ b/codex-rs/analytics/src/analytics_client_tests.rs @@ -19,6 +19,15 @@ use crate::facts::AppInvocation; use crate::facts::AppMentionedInput; use crate::facts::AppUsedInput; use crate::facts::CustomAnalyticsFact; +use crate::facts::GuardianReviewDecision; +use crate::facts::GuardianReviewEventParams; +use crate::facts::GuardianReviewFailureKind; +use crate::facts::GuardianReviewRiskLevel; +use crate::facts::GuardianReviewSessionKind; +use crate::facts::GuardianReviewTerminalStatus; +use crate::facts::GuardianReviewTrigger; +use crate::facts::GuardianReviewedAction; +use crate::facts::GuardianToolCallCounts; use crate::facts::InvocationType; use crate::facts::PluginState; use crate::facts::PluginStateChangedInput; @@ -823,6 +832,127 @@ async fn reducer_ingests_plugin_state_changed_fact() { ); } +#[tokio::test] +async fn reducer_ingests_guardian_review_fact() { + let mut reducer = AnalyticsReducer::default(); + let mut events = Vec::new(); + let tool_counts = GuardianToolCallCounts { + shell: 1, + mcp: 2, + ..Default::default() + }; + + reducer + .ingest( + AnalyticsFact::Custom(CustomAnalyticsFact::GuardianReview(Box::new( + GuardianReviewEventParams { + thread_id: "thread-1".to_string(), + turn_id: "turn-1".to_string(), + review_id: "review-1".to_string(), + target_item_id: "tool-1".to_string(), + product_client_id: Some("codex_app".to_string()), + trigger: GuardianReviewTrigger::McpToolCall, + retry_reason: Some("requires approval".to_string()), + delegated_review: true, + reviewed_action: GuardianReviewedAction::McpToolCall { + server: "github".to_string(), + tool_name: "create_pr".to_string(), + arguments: Some(json!({"title": "Guardian analytics"})), + connector_id: Some("github".to_string()), + connector_name: Some("GitHub".to_string()), + tool_title: Some("Create PR".to_string()), + }, + reviewed_action_truncated: false, + decision: GuardianReviewDecision::Denied, + terminal_status: GuardianReviewTerminalStatus::FailedClosed, + failure_kind: Some(GuardianReviewFailureKind::ParseError), + risk_score: Some(100), + risk_level: Some(GuardianReviewRiskLevel::High), + rationale: Some("Automatic approval review failed".to_string()), + guardian_thread_id: Some("guardian-thread-1".to_string()), + guardian_session_kind: Some(GuardianReviewSessionKind::EphemeralForked), + guardian_model: Some("gpt-5.4".to_string()), + guardian_reasoning_effort: Some("low".to_string()), + had_prior_review_context: Some(true), + review_timeout_ms: 90_000, + guardian_tool_call_count: tool_counts.total(), + guardian_tool_call_counts: tool_counts, + guardian_time_to_first_token_ms: Some(123), + guardian_completion_latency_ms: Some(456), + started_at: 1_716_000_000, + completed_at: Some(1_716_000_001), + input_tokens: Some(10), + cached_input_tokens: Some(2), + output_tokens: Some(3), + reasoning_output_tokens: Some(1), + total_tokens: Some(13), + }, + ))), + &mut events, + ) + .await; + + let payload = serde_json::to_value(&events).expect("serialize guardian review event"); + assert_eq!( + payload, + json!([{ + "event_type": "codex_guardian_review", + "event_params": { + "thread_id": "thread-1", + "turn_id": "turn-1", + "review_id": "review-1", + "target_item_id": "tool-1", + "product_client_id": "codex_app", + "trigger": "mcp_tool_call", + "retry_reason": "requires approval", + "delegated_review": true, + "reviewed_action": { + "type": "mcp_tool_call", + "server": "github", + "tool_name": "create_pr", + "arguments": {"title": "Guardian analytics"}, + "connector_id": "github", + "connector_name": "GitHub", + "tool_title": "Create PR" + }, + "reviewed_action_truncated": false, + "decision": "denied", + "terminal_status": "failed_closed", + "failure_kind": "parse_error", + "risk_score": 100, + "risk_level": "high", + "rationale": "Automatic approval review failed", + "guardian_thread_id": "guardian-thread-1", + "guardian_session_kind": "ephemeral_forked", + "guardian_model": "gpt-5.4", + "guardian_reasoning_effort": "low", + "had_prior_review_context": true, + "review_timeout_ms": 90000, + "guardian_tool_call_count": 3, + "guardian_tool_call_counts": { + "shell": 1, + "unified_exec": 0, + "mcp": 2, + "dynamic": 0, + "apply_patch": 0, + "web_search": 0, + "image_generation": 0, + "view_image": 0 + }, + "guardian_time_to_first_token_ms": 123, + "guardian_completion_latency_ms": 456, + "started_at": 1716000000, + "completed_at": 1716000001, + "input_tokens": 10, + "cached_input_tokens": 2, + "output_tokens": 3, + "reasoning_output_tokens": 1, + "total_tokens": 13 + } + }]) + ); +} + fn sample_plugin_metadata() -> PluginTelemetryMetadata { PluginTelemetryMetadata { plugin_id: PluginId::parse("sample@test").expect("valid plugin id"), diff --git a/codex-rs/analytics/src/client.rs b/codex-rs/analytics/src/client.rs index dd300a7bd6..acdcaea62f 100644 --- a/codex-rs/analytics/src/client.rs +++ b/codex-rs/analytics/src/client.rs @@ -7,6 +7,7 @@ use crate::facts::AppInvocation; use crate::facts::AppMentionedInput; use crate::facts::AppUsedInput; use crate::facts::CustomAnalyticsFact; +use crate::facts::GuardianReviewEventParams; use crate::facts::PluginState; use crate::facts::PluginStateChangedInput; use crate::facts::SkillInvocation; @@ -151,6 +152,12 @@ impl AnalyticsEventsClient { )); } + pub fn track_guardian_review(&self, input: GuardianReviewEventParams) { + self.record_fact(AnalyticsFact::Custom(CustomAnalyticsFact::GuardianReview( + Box::new(input), + ))); + } + pub fn track_app_mentioned(&self, tracking: TrackEventsContext, mentions: Vec) { if mentions.is_empty() { return; diff --git a/codex-rs/analytics/src/events.rs b/codex-rs/analytics/src/events.rs index 885e93bbb9..4206425e84 100644 --- a/codex-rs/analytics/src/events.rs +++ b/codex-rs/analytics/src/events.rs @@ -1,4 +1,5 @@ use crate::facts::AppInvocation; +use crate::facts::GuardianReviewEventParams; use crate::facts::InvocationType; use crate::facts::PluginState; use crate::facts::SubAgentThreadStartedInput; @@ -35,6 +36,7 @@ pub(crate) struct TrackEventsRequest { pub(crate) enum TrackEventRequest { SkillInvocation(SkillInvocationEventRequest), ThreadInitialized(ThreadInitializedEvent), + GuardianReview(Box), AppMentioned(CodexAppMentionedEventRequest), AppUsed(CodexAppUsedEventRequest), PluginUsed(CodexPluginUsedEventRequest), @@ -99,6 +101,12 @@ pub(crate) struct ThreadInitializedEvent { pub(crate) event_params: ThreadInitializedEventParams, } +#[derive(Serialize)] +pub(crate) struct GuardianReviewEventRequest { + pub(crate) event_type: &'static str, + pub(crate) event_params: GuardianReviewEventParams, +} + #[derive(Serialize)] pub(crate) struct CodexAppMetadata { pub(crate) connector_id: Option, diff --git a/codex-rs/analytics/src/facts.rs b/codex-rs/analytics/src/facts.rs index e19d15d847..54a0a98e2f 100644 --- a/codex-rs/analytics/src/facts.rs +++ b/codex-rs/analytics/src/facts.rs @@ -6,6 +6,9 @@ use codex_app_server_protocol::InitializeParams; use codex_app_server_protocol::RequestId; use codex_app_server_protocol::ServerNotification; use codex_plugin::PluginTelemetryMetadata; +use codex_protocol::approvals::NetworkApprovalProtocol; +use codex_protocol::models::PermissionProfile; +use codex_protocol::models::SandboxPermissions; use codex_protocol::protocol::SkillScope; use codex_protocol::protocol::SubAgentSource; use serde::Serialize; @@ -89,6 +92,7 @@ pub(crate) enum AnalyticsFact { pub(crate) enum CustomAnalyticsFact { SubAgentThreadStarted(SubAgentThreadStartedInput), + GuardianReview(Box), SkillInvoked(SkillInvokedInput), AppMentioned(AppMentionedInput), AppUsed(AppUsedInput), @@ -128,3 +132,175 @@ pub(crate) enum PluginState { Enabled, Disabled, } + +#[derive(Clone, Copy, Debug, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum GuardianReviewTrigger { + Shell, + UnifiedExec, + Execve, + ApplyPatch, + NetworkAccess, + McpToolCall, +} + +#[derive(Clone, Copy, Debug, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum GuardianReviewDecision { + Approved, + Denied, + Aborted, +} + +#[derive(Clone, Copy, Debug, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum GuardianReviewTerminalStatus { + Approved, + Denied, + Aborted, + TimedOut, + FailedClosed, +} + +#[derive(Clone, Copy, Debug, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum GuardianReviewFailureKind { + Timeout, + Cancelled, + PromptBuildError, + SessionError, + ParseError, +} + +#[derive(Clone, Copy, Debug, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum GuardianReviewSessionKind { + TrunkSpawned, + TrunkReused, + EphemeralForked, +} + +#[derive(Clone, Copy, Debug, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum GuardianReviewRiskLevel { + Low, + Medium, + High, +} + +#[derive(Clone, Debug, Default, Serialize)] +pub struct GuardianToolCallCounts { + pub shell: u64, + pub unified_exec: u64, + pub mcp: u64, + pub dynamic: u64, + pub apply_patch: u64, + pub web_search: u64, + pub image_generation: u64, + pub view_image: u64, +} + +impl GuardianToolCallCounts { + pub fn total(&self) -> u64 { + self.shell + + self.unified_exec + + self.mcp + + self.dynamic + + self.apply_patch + + self.web_search + + self.image_generation + + self.view_image + } +} + +#[derive(Clone, Debug, Serialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum GuardianReviewedAction { + Shell { + command: Vec, + command_display: String, + cwd: String, + sandbox_permissions: SandboxPermissions, + additional_permissions: Option, + justification: Option, + }, + UnifiedExec { + command: Vec, + command_display: String, + cwd: String, + sandbox_permissions: SandboxPermissions, + additional_permissions: Option, + justification: Option, + tty: bool, + }, + Execve { + source: GuardianCommandSource, + program: String, + argv: Vec, + cwd: String, + additional_permissions: Option, + }, + ApplyPatch { + cwd: String, + files: Vec, + patch: Option, + }, + NetworkAccess { + target: String, + host: String, + protocol: NetworkApprovalProtocol, + port: u16, + }, + McpToolCall { + server: String, + tool_name: String, + arguments: Option, + connector_id: Option, + connector_name: Option, + tool_title: Option, + }, +} + +#[derive(Clone, Copy, Debug, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum GuardianCommandSource { + Shell, + UnifiedExec, +} + +#[derive(Clone, Debug, Serialize)] +pub struct GuardianReviewEventParams { + pub thread_id: String, + pub turn_id: String, + pub review_id: String, + pub target_item_id: String, + pub product_client_id: Option, + pub trigger: GuardianReviewTrigger, + pub retry_reason: Option, + pub delegated_review: bool, + pub reviewed_action: GuardianReviewedAction, + pub reviewed_action_truncated: bool, + pub decision: GuardianReviewDecision, + pub terminal_status: GuardianReviewTerminalStatus, + pub failure_kind: Option, + pub risk_score: Option, + pub risk_level: Option, + pub rationale: Option, + pub guardian_thread_id: Option, + pub guardian_session_kind: Option, + pub guardian_model: Option, + pub guardian_reasoning_effort: Option, + pub had_prior_review_context: Option, + pub review_timeout_ms: u64, + pub guardian_tool_call_count: u64, + pub guardian_tool_call_counts: GuardianToolCallCounts, + pub guardian_time_to_first_token_ms: Option, + pub guardian_completion_latency_ms: Option, + pub started_at: u64, + pub completed_at: Option, + pub input_tokens: Option, + pub cached_input_tokens: Option, + pub output_tokens: Option, + pub reasoning_output_tokens: Option, + pub total_tokens: Option, +} diff --git a/codex-rs/analytics/src/lib.rs b/codex-rs/analytics/src/lib.rs index f2f76ca8cf..bba9665f16 100644 --- a/codex-rs/analytics/src/lib.rs +++ b/codex-rs/analytics/src/lib.rs @@ -6,6 +6,16 @@ mod reducer; pub use client::AnalyticsEventsClient; pub use events::AppServerRpcTransport; pub use facts::AppInvocation; +pub use facts::GuardianCommandSource; +pub use facts::GuardianReviewDecision; +pub use facts::GuardianReviewEventParams; +pub use facts::GuardianReviewFailureKind; +pub use facts::GuardianReviewRiskLevel; +pub use facts::GuardianReviewSessionKind; +pub use facts::GuardianReviewTerminalStatus; +pub use facts::GuardianReviewTrigger; +pub use facts::GuardianReviewedAction; +pub use facts::GuardianToolCallCounts; pub use facts::InvocationType; pub use facts::SkillInvocation; pub use facts::SubAgentThreadStartedInput; diff --git a/codex-rs/analytics/src/reducer.rs b/codex-rs/analytics/src/reducer.rs index 63b9c3d5be..429e0fb8f3 100644 --- a/codex-rs/analytics/src/reducer.rs +++ b/codex-rs/analytics/src/reducer.rs @@ -5,6 +5,7 @@ use crate::events::CodexAppUsedEventRequest; use crate::events::CodexPluginEventRequest; use crate::events::CodexPluginUsedEventRequest; use crate::events::CodexRuntimeMetadata; +use crate::events::GuardianReviewEventRequest; use crate::events::SkillInvocationEventParams; use crate::events::SkillInvocationEventRequest; use crate::events::ThreadInitializationMode; @@ -81,6 +82,9 @@ impl AnalyticsReducer { CustomAnalyticsFact::SubAgentThreadStarted(input) => { self.ingest_subagent_thread_started(input, out); } + CustomAnalyticsFact::GuardianReview(input) => { + self.ingest_guardian_review(*input, out); + } CustomAnalyticsFact::SkillInvoked(input) => { self.ingest_skill_invoked(input, out).await; } @@ -135,6 +139,19 @@ impl AnalyticsReducer { )); } + fn ingest_guardian_review( + &mut self, + input: crate::facts::GuardianReviewEventParams, + out: &mut Vec, + ) { + out.push(TrackEventRequest::GuardianReview(Box::new( + GuardianReviewEventRequest { + event_type: "codex_guardian_review", + event_params: input, + }, + ))); + } + async fn ingest_skill_invoked( &mut self, input: SkillInvokedInput,