Compare commits

...

1 Commits

Author SHA1 Message Date
Mark Steinbrick
50ea3b9636 Add dedicated goal analytics lifecycle event 2026-05-27 15:29:52 -07:00
7 changed files with 341 additions and 0 deletions

View File

@@ -8,6 +8,7 @@ use crate::events::CodexAppUsedEventRequest;
use crate::events::CodexCommandExecutionEventParams;
use crate::events::CodexCommandExecutionEventRequest;
use crate::events::CodexCompactionEventRequest;
use crate::events::CodexGoalEventRequest;
use crate::events::CodexHookRunEventRequest;
use crate::events::CodexPluginEventRequest;
use crate::events::CodexPluginUsedEventRequest;
@@ -43,6 +44,7 @@ use crate::facts::AppInvocation;
use crate::facts::AppMentionedInput;
use crate::facts::AppUsedInput;
use crate::facts::CodexCompactionEvent;
use crate::facts::CodexGoalEvent;
use crate::facts::CompactionImplementation;
use crate::facts::CompactionPhase;
use crate::facts::CompactionReason;
@@ -50,6 +52,9 @@ use crate::facts::CompactionStatus;
use crate::facts::CompactionStrategy;
use crate::facts::CompactionTrigger;
use crate::facts::CustomAnalyticsFact;
use crate::facts::GoalEventKind;
use crate::facts::GoalEventSource;
use crate::facts::GoalStatus;
use crate::facts::HookRunFact;
use crate::facts::HookRunInput;
use crate::facts::InputError;
@@ -1265,6 +1270,63 @@ fn compaction_event_serializes_expected_shape() {
);
}
#[test]
fn goal_event_serializes_expected_shape() {
let event = TrackEventRequest::Goal(Box::new(CodexGoalEventRequest {
event_type: "codex_goal_event",
event_params: crate::events::codex_goal_event_params(
CodexGoalEvent {
thread_id: "thread-1".to_string(),
turn_id: None,
goal_event_kind: GoalEventKind::RuntimeStatusChanged,
goal_event_source: GoalEventSource::RuntimeAccounting,
goal_status: GoalStatus::BudgetLimited,
previous_goal_status: Some(GoalStatus::Active),
},
"session-thread-1".to_string(),
sample_app_server_client_metadata(),
sample_runtime_metadata(),
Some(ThreadSource::User),
/*subagent_source*/ None,
/*parent_thread_id*/ None,
),
}));
let payload = serde_json::to_value(&event).expect("serialize goal event");
assert_eq!(
payload,
json!({
"event_type": "codex_goal_event",
"event_params": {
"thread_id": "thread-1",
"session_id": "session-thread-1",
"turn_id": null,
"app_server_client": {
"product_client_id": DEFAULT_ORIGINATOR,
"client_name": "codex-tui",
"client_version": "1.0.0",
"rpc_transport": "stdio",
"experimental_api_enabled": true
},
"runtime": {
"codex_rs_version": "0.1.0",
"runtime_os": "macos",
"runtime_os_version": "15.3.1",
"runtime_arch": "aarch64"
},
"thread_source": "user",
"subagent_source": null,
"parent_thread_id": null,
"goal_event_kind": "runtime_status_changed",
"goal_event_source": "runtime_accounting",
"goal_status": "budget_limited",
"previous_goal_status": "active"
}
})
);
}
#[test]
fn compaction_implementation_serializes_remote_v2() {
let payload = serde_json::to_value(CompactionImplementation::ResponsesCompactionV2)
@@ -1822,6 +1884,77 @@ async fn compaction_event_ingests_custom_fact() {
assert_eq!(payload[0]["event_params"]["status"], "failed");
}
#[tokio::test]
async fn goal_event_ingests_custom_fact_with_turn_attribution() {
let mut reducer = AnalyticsReducer::default();
let mut events = Vec::new();
ingest_initialize(&mut reducer, &mut events).await;
reducer
.ingest(
AnalyticsFact::ClientResponse {
connection_id: 7,
request_id: RequestId::Integer(2),
response: Box::new(sample_thread_resume_response(
"thread-1", /*ephemeral*/ false, "gpt-5",
)),
},
&mut events,
)
.await;
events.clear();
reducer
.ingest(
AnalyticsFact::Custom(CustomAnalyticsFact::Goal(Box::new(CodexGoalEvent {
thread_id: "thread-1".to_string(),
turn_id: Some("turn-goal".to_string()),
goal_event_kind: GoalEventKind::Created,
goal_event_source: GoalEventSource::GoalTool,
goal_status: GoalStatus::Active,
previous_goal_status: None,
}))),
&mut events,
)
.await;
let payload = serde_json::to_value(&events).expect("serialize events");
assert_eq!(payload.as_array().expect("events array").len(), 1);
assert_eq!(payload[0]["event_type"], "codex_goal_event");
assert_eq!(payload[0]["event_params"]["session_id"], "session-thread-1");
assert_eq!(payload[0]["event_params"]["thread_id"], "thread-1");
assert_eq!(payload[0]["event_params"]["turn_id"], "turn-goal");
assert_eq!(payload[0]["event_params"]["goal_event_kind"], "created");
assert_eq!(payload[0]["event_params"]["goal_event_source"], "goal_tool");
assert_eq!(payload[0]["event_params"]["goal_status"], "active");
assert_eq!(
payload[0]["event_params"]["previous_goal_status"],
json!(null)
);
}
#[tokio::test]
async fn goal_event_without_thread_context_is_dropped() {
let mut reducer = AnalyticsReducer::default();
let mut events = Vec::new();
reducer
.ingest(
AnalyticsFact::Custom(CustomAnalyticsFact::Goal(Box::new(CodexGoalEvent {
thread_id: "thread-missing".to_string(),
turn_id: Some("turn-missing".to_string()),
goal_event_kind: GoalEventKind::Created,
goal_event_source: GoalEventSource::GoalTool,
goal_status: GoalStatus::Active,
previous_goal_status: None,
}))),
&mut events,
)
.await;
assert!(events.is_empty());
}
#[tokio::test]
async fn guardian_review_event_ingests_custom_fact_with_optional_target_item() {
let mut reducer = AnalyticsReducer::default();

View File

@@ -9,6 +9,7 @@ use crate::facts::AnalyticsJsonRpcError;
use crate::facts::AppInvocation;
use crate::facts::AppMentionedInput;
use crate::facts::AppUsedInput;
use crate::facts::CodexGoalEvent;
use crate::facts::CustomAnalyticsFact;
use crate::facts::HookRunFact;
use crate::facts::HookRunInput;
@@ -244,6 +245,12 @@ impl AnalyticsEventsClient {
)));
}
pub fn track_goal_event(&self, event: CodexGoalEvent) {
self.record_fact(AnalyticsFact::Custom(CustomAnalyticsFact::Goal(Box::new(
event,
))));
}
pub fn track_turn_resolved_config(&self, fact: TurnResolvedConfigFact) {
self.record_fact(AnalyticsFact::Custom(
CustomAnalyticsFact::TurnResolvedConfig(Box::new(fact)),

View File

@@ -3,12 +3,16 @@ use std::time::Instant;
use crate::facts::AcceptedLineFingerprint;
use crate::facts::AppInvocation;
use crate::facts::CodexCompactionEvent;
use crate::facts::CodexGoalEvent;
use crate::facts::CompactionImplementation;
use crate::facts::CompactionPhase;
use crate::facts::CompactionReason;
use crate::facts::CompactionStatus;
use crate::facts::CompactionStrategy;
use crate::facts::CompactionTrigger;
use crate::facts::GoalEventKind;
use crate::facts::GoalEventSource;
use crate::facts::GoalStatus;
use crate::facts::HookRunFact;
use crate::facts::InvocationType;
use crate::facts::PluginState;
@@ -62,6 +66,7 @@ pub(crate) enum TrackEventRequest {
AppUsed(CodexAppUsedEventRequest),
HookRun(CodexHookRunEventRequest),
Compaction(Box<CodexCompactionEventRequest>),
Goal(Box<CodexGoalEventRequest>),
TurnEvent(Box<CodexTurnEventRequest>),
TurnSteer(CodexTurnSteerEventRequest),
CommandExecution(CodexCommandExecutionEventRequest),
@@ -767,6 +772,28 @@ pub(crate) struct CodexCompactionEventRequest {
pub(crate) event_params: CodexCompactionEventParams,
}
#[derive(Serialize)]
pub(crate) struct CodexGoalEventParams {
pub(crate) thread_id: String,
pub(crate) session_id: String,
pub(crate) turn_id: Option<String>,
pub(crate) app_server_client: CodexAppServerClientMetadata,
pub(crate) runtime: CodexRuntimeMetadata,
pub(crate) thread_source: Option<ThreadSource>,
pub(crate) subagent_source: Option<String>,
pub(crate) parent_thread_id: Option<String>,
pub(crate) goal_event_kind: GoalEventKind,
pub(crate) goal_event_source: GoalEventSource,
pub(crate) goal_status: GoalStatus,
pub(crate) previous_goal_status: Option<GoalStatus>,
}
#[derive(Serialize)]
pub(crate) struct CodexGoalEventRequest {
pub(crate) event_type: &'static str,
pub(crate) event_params: CodexGoalEventParams,
}
#[derive(Serialize)]
pub(crate) struct CodexTurnEventParams {
pub(crate) thread_id: String,
@@ -962,6 +989,31 @@ pub(crate) fn codex_compaction_event_params(
}
}
pub(crate) fn codex_goal_event_params(
input: CodexGoalEvent,
session_id: String,
app_server_client: CodexAppServerClientMetadata,
runtime: CodexRuntimeMetadata,
thread_source: Option<ThreadSource>,
subagent_source: Option<String>,
parent_thread_id: Option<String>,
) -> CodexGoalEventParams {
CodexGoalEventParams {
thread_id: input.thread_id,
session_id,
turn_id: input.turn_id,
app_server_client,
runtime,
thread_source,
subagent_source,
parent_thread_id,
goal_event_kind: input.goal_event_kind,
goal_event_source: input.goal_event_source,
goal_status: input.goal_status,
previous_goal_status: input.previous_goal_status,
}
}
pub(crate) fn codex_plugin_used_metadata(
tracking: &TrackEventsContext,
plugin: PluginTelemetryMetadata,

View File

@@ -275,6 +275,43 @@ pub struct CodexCompactionEvent {
pub duration_ms: Option<u64>,
}
#[derive(Clone, Copy, Debug, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum GoalEventKind {
Created,
Updated,
RuntimeStatusChanged,
}
#[derive(Clone, Copy, Debug, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum GoalEventSource {
GoalTool,
RuntimeAccounting,
UsageLimit,
}
#[derive(Clone, Copy, Debug, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum GoalStatus {
Active,
Paused,
Blocked,
UsageLimited,
BudgetLimited,
Complete,
}
#[derive(Clone)]
pub struct CodexGoalEvent {
pub thread_id: String,
pub turn_id: Option<String>,
pub goal_event_kind: GoalEventKind,
pub goal_event_source: GoalEventSource,
pub goal_status: GoalStatus,
pub previous_goal_status: Option<GoalStatus>,
}
#[allow(dead_code)]
pub(crate) enum AnalyticsFact {
Initialize {
@@ -326,6 +363,7 @@ pub(crate) enum AnalyticsFact {
pub(crate) enum CustomAnalyticsFact {
SubAgentThreadStarted(SubAgentThreadStartedInput),
Compaction(Box<CodexCompactionEvent>),
Goal(Box<CodexGoalEvent>),
GuardianReview(Box<GuardianReviewEventParams>),
TurnResolvedConfig(Box<TurnResolvedConfigFact>),
TurnTokenUsage(Box<TurnTokenUsageFact>),

View File

@@ -24,6 +24,7 @@ pub use facts::AcceptedLineFingerprint;
pub use facts::AnalyticsJsonRpcError;
pub use facts::AppInvocation;
pub use facts::CodexCompactionEvent;
pub use facts::CodexGoalEvent;
pub use facts::CodexTurnSteerEvent;
pub use facts::CompactionImplementation;
pub use facts::CompactionPhase;
@@ -31,6 +32,9 @@ pub use facts::CompactionReason;
pub use facts::CompactionStatus;
pub use facts::CompactionStrategy;
pub use facts::CompactionTrigger;
pub use facts::GoalEventKind;
pub use facts::GoalEventSource;
pub use facts::GoalStatus;
pub use facts::HookRunFact;
pub use facts::InputError;
pub use facts::InvocationType;

View File

@@ -15,6 +15,7 @@ use crate::events::CodexDynamicToolCallEventParams;
use crate::events::CodexDynamicToolCallEventRequest;
use crate::events::CodexFileChangeEventParams;
use crate::events::CodexFileChangeEventRequest;
use crate::events::CodexGoalEventRequest;
use crate::events::CodexHookRunEventRequest;
use crate::events::CodexImageGenerationEventParams;
use crate::events::CodexImageGenerationEventRequest;
@@ -51,6 +52,7 @@ use crate::events::TrackEventRequest;
use crate::events::WebSearchActionKind;
use crate::events::codex_app_metadata;
use crate::events::codex_compaction_event_params;
use crate::events::codex_goal_event_params;
use crate::events::codex_hook_run_metadata;
use crate::events::codex_plugin_metadata;
use crate::events::codex_plugin_used_metadata;
@@ -63,6 +65,7 @@ use crate::facts::AnalyticsJsonRpcError;
use crate::facts::AppMentionedInput;
use crate::facts::AppUsedInput;
use crate::facts::CodexCompactionEvent;
use crate::facts::CodexGoalEvent;
use crate::facts::CustomAnalyticsFact;
use crate::facts::HookRunInput;
use crate::facts::PluginState;
@@ -189,6 +192,16 @@ impl<'a> AnalyticsDropSite<'a> {
}
}
fn goal(input: &'a CodexGoalEvent) -> Self {
Self {
event_name: "goal",
thread_id: &input.thread_id,
turn_id: input.turn_id.as_deref(),
review_id: None,
item_id: None,
}
}
fn tool_item(
notification: &'a codex_app_server_protocol::ItemCompletedNotification,
item_id: &'a str,
@@ -455,6 +468,9 @@ impl AnalyticsReducer {
CustomAnalyticsFact::Compaction(input) => {
self.ingest_compaction(*input, out);
}
CustomAnalyticsFact::Goal(input) => {
self.ingest_goal_event(*input, out);
}
CustomAnalyticsFact::GuardianReview(input) => {
self.ingest_guardian_review(*input, out);
}
@@ -1296,6 +1312,26 @@ impl AnalyticsReducer {
)));
}
fn ingest_goal_event(&mut self, input: CodexGoalEvent, out: &mut Vec<TrackEventRequest>) {
let Some((connection_state, thread_metadata)) =
self.thread_context_or_warn(AnalyticsDropSite::goal(&input))
else {
return;
};
out.push(TrackEventRequest::Goal(Box::new(CodexGoalEventRequest {
event_type: "codex_goal_event",
event_params: codex_goal_event_params(
input,
thread_metadata.session_id.clone(),
connection_state.app_server_client.clone(),
connection_state.runtime.clone(),
thread_metadata.thread_source,
thread_metadata.subagent_source.clone(),
thread_metadata.parent_thread_id.clone(),
),
})));
}
fn ingest_guardian_review_completed(
&mut self,
notification: codex_app_server_protocol::ItemGuardianApprovalReviewCompletedNotification,

View File

@@ -15,6 +15,10 @@ use crate::state::TurnState;
use crate::tasks::RegularTask;
use crate::tools::handlers::goal_spec::UPDATE_GOAL_TOOL_NAME;
use anyhow::Context;
use codex_analytics::CodexGoalEvent;
use codex_analytics::GoalEventKind;
use codex_analytics::GoalEventSource;
use codex_analytics::GoalStatus as AnalyticsGoalStatus;
use codex_features::Feature;
use codex_otel::GOAL_BLOCKED_METRIC;
use codex_otel::GOAL_BUDGET_LIMITED_METRIC;
@@ -549,6 +553,17 @@ impl Session {
}
self.emit_goal_resumed_metric_if_status_changed(previous_status_for_goal, goal_status);
self.emit_goal_terminal_metrics_if_status_changed(previous_status_for_goal, &goal);
self.track_goal_analytics_event(
Some(turn_context.sub_id.clone()),
if replacing_goal {
GoalEventKind::Created
} else {
GoalEventKind::Updated
},
GoalEventSource::GoalTool,
goal_status,
previous_status_for_goal,
);
let goal = protocol_goal_from_state(goal);
*self.goal_runtime.budget_limit_reported_goal_id.lock().await = None;
let newly_active_goal = goal_status == codex_state::ThreadGoalStatus::Active
@@ -626,6 +641,13 @@ impl Session {
.await;
let goal_id = goal.goal_id.clone();
self.emit_goal_created_metric();
self.track_goal_analytics_event(
Some(turn_context.sub_id.clone()),
GoalEventKind::Created,
GoalEventSource::GoalTool,
goal.status,
/*previous_goal_status*/ None,
);
let goal = protocol_goal_from_state(goal);
*self.goal_runtime.budget_limit_reported_goal_id.lock().await = None;
@@ -750,6 +772,26 @@ impl Session {
accounting.wall_clock.mark_active_goal(goal_id);
}
fn track_goal_analytics_event(
&self,
turn_id: Option<String>,
goal_event_kind: GoalEventKind,
goal_event_source: GoalEventSource,
goal_status: codex_state::ThreadGoalStatus,
previous_goal_status: Option<codex_state::ThreadGoalStatus>,
) {
self.services
.analytics_events_client
.track_goal_event(CodexGoalEvent {
thread_id: self.conversation_id.to_string(),
turn_id,
goal_event_kind,
goal_event_source,
goal_status: analytics_goal_status(goal_status),
previous_goal_status: previous_goal_status.map(analytics_goal_status),
});
}
fn emit_goal_created_metric(&self) {
self.services
.session_telemetry
@@ -1053,6 +1095,15 @@ impl Session {
}
codex_state::GoalAccountingOutcome::Unchanged(_) => return Ok(()),
};
if previous_status.is_some_and(|previous_status| previous_status != goal.status) {
self.track_goal_analytics_event(
Some(turn_context.sub_id.clone()),
GoalEventKind::RuntimeStatusChanged,
GoalEventSource::RuntimeAccounting,
goal.status,
previous_status,
);
}
let should_steer_budget_limit =
matches!(budget_limit_steering, BudgetLimitSteering::Allowed)
&& goal.status == codex_state::ThreadGoalStatus::BudgetLimited
@@ -1203,6 +1254,15 @@ impl Session {
return Ok(());
};
self.emit_goal_terminal_metrics_if_status_changed(previous_status, &goal);
if previous_status.is_some_and(|previous_status| previous_status != goal.status) {
self.track_goal_analytics_event(
Some(turn_context.sub_id.clone()),
GoalEventKind::RuntimeStatusChanged,
GoalEventSource::UsageLimit,
goal.status,
previous_status,
);
}
let goal = protocol_goal_from_state(goal);
*self.goal_runtime.budget_limit_reported_goal_id.lock().await = None;
self.clear_active_goal_accounting(turn_context).await;
@@ -1638,6 +1698,17 @@ pub(crate) fn protocol_goal_status_from_state(
}
}
fn analytics_goal_status(status: codex_state::ThreadGoalStatus) -> AnalyticsGoalStatus {
match status {
codex_state::ThreadGoalStatus::Active => AnalyticsGoalStatus::Active,
codex_state::ThreadGoalStatus::Paused => AnalyticsGoalStatus::Paused,
codex_state::ThreadGoalStatus::Blocked => AnalyticsGoalStatus::Blocked,
codex_state::ThreadGoalStatus::UsageLimited => AnalyticsGoalStatus::UsageLimited,
codex_state::ThreadGoalStatus::BudgetLimited => AnalyticsGoalStatus::BudgetLimited,
codex_state::ThreadGoalStatus::Complete => AnalyticsGoalStatus::Complete,
}
}
pub(crate) fn state_goal_status_from_protocol(
status: ThreadGoalStatus,
) -> codex_state::ThreadGoalStatus {