clone/codex
1
0
Fork 0
mirror of https://github.com/openai/codex.git synced 2026-05-03 02:46:39 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(analytics): add guardian review event schema

Browse Source
This commit is contained in:
Owen Lin
2026-04-07 16:06:57 -07:00
parent 59af4a730c
commit 823938ff87
7 changed files with 349 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
codex-rs/analytics/Cargo.toml
View File

@@ -20,6 +20,7 @@ codex-plugin = { workspace = true }
codex-protocol = { workspace = true }
os_info = { workspace = true }
serde = { workspace = true, features = ["derive"] }
serde_json = { workspace = true }
sha1 = { workspace = true }
tokio = { workspace = true, features = [
"macros",
@@ -29,4 +30,3 @@ tracing = { workspace = true, features = ["log"] }
[dev-dependencies]
pretty_assertions = { workspace = true }
serde_json = { workspace = true }

130
codex-rs/analytics/src/analytics_client_tests.rs
View File

@@ -19,6 +19,15 @@ use crate::facts::AppInvocation;
use crate::facts::AppMentionedInput;
use crate::facts::AppUsedInput;
use crate::facts::CustomAnalyticsFact;
use crate::facts::GuardianReviewDecision;
use crate::facts::GuardianReviewEventParams;
use crate::facts::GuardianReviewFailureKind;
use crate::facts::GuardianReviewRiskLevel;
use crate::facts::GuardianReviewSessionKind;
use crate::facts::GuardianReviewTerminalStatus;
use crate::facts::GuardianReviewTrigger;
use crate::facts::GuardianReviewedAction;
use crate::facts::GuardianToolCallCounts;
use crate::facts::InvocationType;
use crate::facts::PluginState;
use crate::facts::PluginStateChangedInput;
@@ -823,6 +832,127 @@ async fn reducer_ingests_plugin_state_changed_fact() {
);
}
#[tokio::test]
async fn reducer_ingests_guardian_review_fact() {
let mut reducer = AnalyticsReducer::default();
let mut events = Vec::new();
let tool_counts = GuardianToolCallCounts {
shell: 1,
mcp: 2,
..Default::default()
};
reducer
.ingest(
AnalyticsFact::Custom(CustomAnalyticsFact::GuardianReview(Box::new(
GuardianReviewEventParams {
thread_id: "thread-1".to_string(),
turn_id: "turn-1".to_string(),
review_id: "review-1".to_string(),
target_item_id: "tool-1".to_string(),
product_client_id: Some("codex_app".to_string()),
trigger: GuardianReviewTrigger::McpToolCall,
retry_reason: Some("requires approval".to_string()),
delegated_review: true,
reviewed_action: GuardianReviewedAction::McpToolCall {
server: "github".to_string(),
tool_name: "create_pr".to_string(),
arguments: Some(json!({"title": "Guardian analytics"})),
connector_id: Some("github".to_string()),
connector_name: Some("GitHub".to_string()),
tool_title: Some("Create PR".to_string()),
},
reviewed_action_truncated: false,
decision: GuardianReviewDecision::Denied,
terminal_status: GuardianReviewTerminalStatus::FailedClosed,
failure_kind: Some(GuardianReviewFailureKind::ParseError),
risk_score: Some(100),
risk_level: Some(GuardianReviewRiskLevel::High),
rationale: Some("Automatic approval review failed".to_string()),
guardian_thread_id: Some("guardian-thread-1".to_string()),
guardian_session_kind: Some(GuardianReviewSessionKind::EphemeralForked),
guardian_model: Some("gpt-5.4".to_string()),
guardian_reasoning_effort: Some("low".to_string()),
had_prior_review_context: Some(true),
review_timeout_ms: 90_000,
guardian_tool_call_count: tool_counts.total(),
guardian_tool_call_counts: tool_counts,
guardian_time_to_first_token_ms: Some(123),
guardian_completion_latency_ms: Some(456),
started_at: 1_716_000_000,
completed_at: Some(1_716_000_001),
input_tokens: Some(10),
cached_input_tokens: Some(2),
output_tokens: Some(3),
reasoning_output_tokens: Some(1),
total_tokens: Some(13),
},
))),
&mut events,
)
.await;
let payload = serde_json::to_value(&events).expect("serialize guardian review event");
assert_eq!(
payload,
json!([{
"event_type": "codex_guardian_review",
"event_params": {
"thread_id": "thread-1",
"turn_id": "turn-1",
"review_id": "review-1",
"target_item_id": "tool-1",
"product_client_id": "codex_app",
"trigger": "mcp_tool_call",
"retry_reason": "requires approval",
"delegated_review": true,
"reviewed_action": {
"type": "mcp_tool_call",
"server": "github",
"tool_name": "create_pr",
"arguments": {"title": "Guardian analytics"},
"connector_id": "github",
"connector_name": "GitHub",
"tool_title": "Create PR"
},
"reviewed_action_truncated": false,
"decision": "denied",
"terminal_status": "failed_closed",
"failure_kind": "parse_error",
"risk_score": 100,
"risk_level": "high",
"rationale": "Automatic approval review failed",
"guardian_thread_id": "guardian-thread-1",
"guardian_session_kind": "ephemeral_forked",
"guardian_model": "gpt-5.4",
"guardian_reasoning_effort": "low",
"had_prior_review_context": true,
"review_timeout_ms": 90000,
"guardian_tool_call_count": 3,
"guardian_tool_call_counts": {
"shell": 1,
"unified_exec": 0,
"mcp": 2,
"dynamic": 0,
"apply_patch": 0,
"web_search": 0,
"image_generation": 0,
"view_image": 0
},
"guardian_time_to_first_token_ms": 123,
"guardian_completion_latency_ms": 456,
"started_at": 1716000000,
"completed_at": 1716000001,
"input_tokens": 10,
"cached_input_tokens": 2,
"output_tokens": 3,
"reasoning_output_tokens": 1,
"total_tokens": 13
}
}])
);
}
fn sample_plugin_metadata() -> PluginTelemetryMetadata {
PluginTelemetryMetadata {
plugin_id: PluginId::parse("sample@test").expect("valid plugin id"),

7
codex-rs/analytics/src/client.rs
View File

@@ -7,6 +7,7 @@ use crate::facts::AppInvocation;
use crate::facts::AppMentionedInput;
use crate::facts::AppUsedInput;
use crate::facts::CustomAnalyticsFact;
use crate::facts::GuardianReviewEventParams;
use crate::facts::PluginState;
use crate::facts::PluginStateChangedInput;
use crate::facts::SkillInvocation;
@@ -151,6 +152,12 @@ impl AnalyticsEventsClient {
));
}
pub fn track_guardian_review(&self, input: GuardianReviewEventParams) {
self.record_fact(AnalyticsFact::Custom(CustomAnalyticsFact::GuardianReview(
Box::new(input),
)));
}
pub fn track_app_mentioned(&self, tracking: TrackEventsContext, mentions: Vec<AppInvocation>) {
if mentions.is_empty() {
return;

8
codex-rs/analytics/src/events.rs
View File

@@ -1,4 +1,5 @@
use crate::facts::AppInvocation;
use crate::facts::GuardianReviewEventParams;
use crate::facts::InvocationType;
use crate::facts::PluginState;
use crate::facts::SubAgentThreadStartedInput;
@@ -35,6 +36,7 @@ pub(crate) struct TrackEventsRequest {
pub(crate) enum TrackEventRequest {
SkillInvocation(SkillInvocationEventRequest),
ThreadInitialized(ThreadInitializedEvent),
GuardianReview(Box<GuardianReviewEventRequest>),
AppMentioned(CodexAppMentionedEventRequest),
AppUsed(CodexAppUsedEventRequest),
PluginUsed(CodexPluginUsedEventRequest),
@@ -99,6 +101,12 @@ pub(crate) struct ThreadInitializedEvent {
pub(crate) event_params: ThreadInitializedEventParams,
}
#[derive(Serialize)]
pub(crate) struct GuardianReviewEventRequest {
pub(crate) event_type: &'static str,
pub(crate) event_params: GuardianReviewEventParams,
}
#[derive(Serialize)]
pub(crate) struct CodexAppMetadata {
pub(crate) connector_id: Option<String>,

176
codex-rs/analytics/src/facts.rs
View File

@@ -6,6 +6,9 @@ use codex_app_server_protocol::InitializeParams;
use codex_app_server_protocol::RequestId;
use codex_app_server_protocol::ServerNotification;
use codex_plugin::PluginTelemetryMetadata;
use codex_protocol::approvals::NetworkApprovalProtocol;
use codex_protocol::models::PermissionProfile;
use codex_protocol::models::SandboxPermissions;
use codex_protocol::protocol::SkillScope;
use codex_protocol::protocol::SubAgentSource;
use serde::Serialize;
@@ -89,6 +92,7 @@ pub(crate) enum AnalyticsFact {
pub(crate) enum CustomAnalyticsFact {
SubAgentThreadStarted(SubAgentThreadStartedInput),
GuardianReview(Box<GuardianReviewEventParams>),
SkillInvoked(SkillInvokedInput),
AppMentioned(AppMentionedInput),
AppUsed(AppUsedInput),
@@ -128,3 +132,175 @@ pub(crate) enum PluginState {
Enabled,
Disabled,
}
#[derive(Clone, Copy, Debug, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum GuardianReviewTrigger {
Shell,
UnifiedExec,
Execve,
ApplyPatch,
NetworkAccess,
McpToolCall,
}
#[derive(Clone, Copy, Debug, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum GuardianReviewDecision {
Approved,
Denied,
Aborted,
}
#[derive(Clone, Copy, Debug, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum GuardianReviewTerminalStatus {
Approved,
Denied,
Aborted,
TimedOut,
FailedClosed,
}
#[derive(Clone, Copy, Debug, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum GuardianReviewFailureKind {
Timeout,
Cancelled,
PromptBuildError,
SessionError,
ParseError,
}
#[derive(Clone, Copy, Debug, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum GuardianReviewSessionKind {
TrunkSpawned,
TrunkReused,
EphemeralForked,
}
#[derive(Clone, Copy, Debug, Serialize)]
#[serde(rename_all = "lowercase")]
pub enum GuardianReviewRiskLevel {
Low,
Medium,
High,
}
#[derive(Clone, Debug, Default, Serialize)]
pub struct GuardianToolCallCounts {
pub shell: u64,
pub unified_exec: u64,
pub mcp: u64,
pub dynamic: u64,
pub apply_patch: u64,
pub web_search: u64,
pub image_generation: u64,
pub view_image: u64,
}
impl GuardianToolCallCounts {
pub fn total(&self) -> u64 {
self.shell
+ self.unified_exec
+ self.mcp
+ self.dynamic
+ self.apply_patch
+ self.web_search
+ self.image_generation
+ self.view_image
}
}
#[derive(Clone, Debug, Serialize)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum GuardianReviewedAction {
Shell {
command: Vec<String>,
command_display: String,
cwd: String,
sandbox_permissions: SandboxPermissions,
additional_permissions: Option<PermissionProfile>,
justification: Option<String>,
},
UnifiedExec {
command: Vec<String>,
command_display: String,
cwd: String,
sandbox_permissions: SandboxPermissions,
additional_permissions: Option<PermissionProfile>,
justification: Option<String>,
tty: bool,
},
Execve {
source: GuardianCommandSource,
program: String,
argv: Vec<String>,
cwd: String,
additional_permissions: Option<PermissionProfile>,
},
ApplyPatch {
cwd: String,
files: Vec<String>,
patch: Option<String>,
},
NetworkAccess {
target: String,
host: String,
protocol: NetworkApprovalProtocol,
port: u16,
},
McpToolCall {
server: String,
tool_name: String,
arguments: Option<serde_json::Value>,
connector_id: Option<String>,
connector_name: Option<String>,
tool_title: Option<String>,
},
}
#[derive(Clone, Copy, Debug, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum GuardianCommandSource {
Shell,
UnifiedExec,
}
#[derive(Clone, Debug, Serialize)]
pub struct GuardianReviewEventParams {
pub thread_id: String,
pub turn_id: String,
pub review_id: String,
pub target_item_id: String,
pub product_client_id: Option<String>,
pub trigger: GuardianReviewTrigger,
pub retry_reason: Option<String>,
pub delegated_review: bool,
pub reviewed_action: GuardianReviewedAction,
pub reviewed_action_truncated: bool,
pub decision: GuardianReviewDecision,
pub terminal_status: GuardianReviewTerminalStatus,
pub failure_kind: Option<GuardianReviewFailureKind>,
pub risk_score: Option<u8>,
pub risk_level: Option<GuardianReviewRiskLevel>,
pub rationale: Option<String>,
pub guardian_thread_id: Option<String>,
pub guardian_session_kind: Option<GuardianReviewSessionKind>,
pub guardian_model: Option<String>,
pub guardian_reasoning_effort: Option<String>,
pub had_prior_review_context: Option<bool>,
pub review_timeout_ms: u64,
pub guardian_tool_call_count: u64,
pub guardian_tool_call_counts: GuardianToolCallCounts,
pub guardian_time_to_first_token_ms: Option<u64>,
pub guardian_completion_latency_ms: Option<u64>,
pub started_at: u64,
pub completed_at: Option<u64>,
pub input_tokens: Option<i64>,
pub cached_input_tokens: Option<i64>,
pub output_tokens: Option<i64>,
pub reasoning_output_tokens: Option<i64>,
pub total_tokens: Option<i64>,
}

10
codex-rs/analytics/src/lib.rs
View File

@@ -6,6 +6,16 @@ mod reducer;
pub use client::AnalyticsEventsClient;
pub use events::AppServerRpcTransport;
pub use facts::AppInvocation;
pub use facts::GuardianCommandSource;
pub use facts::GuardianReviewDecision;
pub use facts::GuardianReviewEventParams;
pub use facts::GuardianReviewFailureKind;
pub use facts::GuardianReviewRiskLevel;
pub use facts::GuardianReviewSessionKind;
pub use facts::GuardianReviewTerminalStatus;
pub use facts::GuardianReviewTrigger;
pub use facts::GuardianReviewedAction;
pub use facts::GuardianToolCallCounts;
pub use facts::InvocationType;
pub use facts::SkillInvocation;
pub use facts::SubAgentThreadStartedInput;

17
codex-rs/analytics/src/reducer.rs
View File

@@ -5,6 +5,7 @@ use crate::events::CodexAppUsedEventRequest;
use crate::events::CodexPluginEventRequest;
use crate::events::CodexPluginUsedEventRequest;
use crate::events::CodexRuntimeMetadata;
use crate::events::GuardianReviewEventRequest;
use crate::events::SkillInvocationEventParams;
use crate::events::SkillInvocationEventRequest;
use crate::events::ThreadInitializationMode;
@@ -81,6 +82,9 @@ impl AnalyticsReducer {
CustomAnalyticsFact::SubAgentThreadStarted(input) => {
self.ingest_subagent_thread_started(input, out);
}
CustomAnalyticsFact::GuardianReview(input) => {
self.ingest_guardian_review(*input, out);
}
CustomAnalyticsFact::SkillInvoked(input) => {
self.ingest_skill_invoked(input, out).await;
}
@@ -135,6 +139,19 @@ impl AnalyticsReducer {
));
}
fn ingest_guardian_review(
&mut self,
input: crate::facts::GuardianReviewEventParams,
out: &mut Vec<TrackEventRequest>,
) {
out.push(TrackEventRequest::GuardianReview(Box::new(
GuardianReviewEventRequest {
event_type: "codex_guardian_review",
event_params: input,
},
)));
}
async fn ingest_skill_invoked(
&mut self,
input: SkillInvokedInput,