Files
codex/codex-rs/core/src/hook_runtime.rs
Abhinav 8720b7bdce Add codex_hook_run analytics event (#17996)
# Why
Add product analytics for hook handler executions so we can understand
which hooks are running, where they came from, and whether they
completed, failed, stopped, or blocked work.

# What
- add the new `codex_hook_run` analytics event and payload plumbing in
`codex-rs/analytics`
- emit hook-run analytics from the shared hook completion path in
`codex-rs/core`
- classify hook source from the loaded hook path as `system`, `user`,
`project`, or `unknown`

```
{
  "event_type": "codex_hook_run",
  "event_params": {
    "thread_id": "string",
    "turn_id": "string",
    "model_slug": "string",
    "hook_name": "string, // any HookEventName
    "hook_source": "system | user | project | unknown",
    "status": "completed | failed | stopped | blocked"
  }
}
```

---------

Co-authored-by: Codex <noreply@openai.com>
2026-04-16 19:43:16 +00:00

485 lines
16 KiB
Rust

use std::future::Future;
use std::sync::Arc;
use codex_analytics::HookRunFact;
use codex_analytics::build_track_events_context;
use codex_hooks::PostToolUseOutcome;
use codex_hooks::PostToolUseRequest;
use codex_hooks::PreToolUseOutcome;
use codex_hooks::PreToolUseRequest;
use codex_hooks::SessionStartOutcome;
use codex_hooks::UserPromptSubmitOutcome;
use codex_hooks::UserPromptSubmitRequest;
use codex_protocol::items::TurnItem;
use codex_protocol::models::DeveloperInstructions;
use codex_protocol::models::ResponseInputItem;
use codex_protocol::models::ResponseItem;
use codex_protocol::protocol::AskForApproval;
use codex_protocol::protocol::EventMsg;
use codex_protocol::protocol::HookCompletedEvent;
use codex_protocol::protocol::HookRunSummary;
use codex_protocol::protocol::HookStartedEvent;
use codex_protocol::user_input::UserInput;
use serde_json::Value;
use crate::codex::Session;
use crate::codex::TurnContext;
use crate::event_mapping::parse_turn_item;
pub(crate) struct HookRuntimeOutcome {
pub should_stop: bool,
pub additional_contexts: Vec<String>,
}
pub(crate) enum PendingInputHookDisposition {
Accepted(Box<PendingInputRecord>),
Blocked { additional_contexts: Vec<String> },
}
pub(crate) enum PendingInputRecord {
UserMessage {
content: Vec<UserInput>,
response_item: ResponseItem,
additional_contexts: Vec<String>,
},
ConversationItem {
response_item: ResponseItem,
},
}
struct ContextInjectingHookOutcome {
hook_events: Vec<HookCompletedEvent>,
outcome: HookRuntimeOutcome,
}
impl From<SessionStartOutcome> for ContextInjectingHookOutcome {
fn from(value: SessionStartOutcome) -> Self {
let SessionStartOutcome {
hook_events,
should_stop,
stop_reason: _,
additional_contexts,
} = value;
Self {
hook_events,
outcome: HookRuntimeOutcome {
should_stop,
additional_contexts,
},
}
}
}
impl From<UserPromptSubmitOutcome> for ContextInjectingHookOutcome {
fn from(value: UserPromptSubmitOutcome) -> Self {
let UserPromptSubmitOutcome {
hook_events,
should_stop,
stop_reason: _,
additional_contexts,
} = value;
Self {
hook_events,
outcome: HookRuntimeOutcome {
should_stop,
additional_contexts,
},
}
}
}
pub(crate) async fn run_pending_session_start_hooks(
sess: &Arc<Session>,
turn_context: &Arc<TurnContext>,
) -> bool {
let Some(session_start_source) = sess.take_pending_session_start_source().await else {
return false;
};
let request = codex_hooks::SessionStartRequest {
session_id: sess.conversation_id,
cwd: turn_context.cwd.clone(),
transcript_path: sess.hook_transcript_path().await,
model: turn_context.model_info.slug.clone(),
permission_mode: hook_permission_mode(turn_context),
source: session_start_source,
};
let preview_runs = sess.hooks().preview_session_start(&request);
run_context_injecting_hook(
sess,
turn_context,
preview_runs,
sess.hooks()
.run_session_start(request, Some(turn_context.sub_id.clone())),
)
.await
.record_additional_contexts(sess, turn_context)
.await
}
pub(crate) async fn run_pre_tool_use_hooks(
sess: &Arc<Session>,
turn_context: &Arc<TurnContext>,
tool_use_id: String,
command: String,
) -> Option<String> {
let request = PreToolUseRequest {
session_id: sess.conversation_id,
turn_id: turn_context.sub_id.clone(),
cwd: turn_context.cwd.clone(),
transcript_path: sess.hook_transcript_path().await,
model: turn_context.model_info.slug.clone(),
permission_mode: hook_permission_mode(turn_context),
tool_name: "Bash".to_string(),
tool_use_id,
command,
};
let preview_runs = sess.hooks().preview_pre_tool_use(&request);
emit_hook_started_events(sess, turn_context, preview_runs).await;
let PreToolUseOutcome {
hook_events,
should_block,
block_reason,
} = sess.hooks().run_pre_tool_use(request).await;
emit_hook_completed_events(sess, turn_context, hook_events).await;
if should_block { block_reason } else { None }
}
pub(crate) async fn run_post_tool_use_hooks(
sess: &Arc<Session>,
turn_context: &Arc<TurnContext>,
tool_use_id: String,
command: String,
tool_response: Value,
) -> PostToolUseOutcome {
let request = PostToolUseRequest {
session_id: sess.conversation_id,
turn_id: turn_context.sub_id.clone(),
cwd: turn_context.cwd.clone(),
transcript_path: sess.hook_transcript_path().await,
model: turn_context.model_info.slug.clone(),
permission_mode: hook_permission_mode(turn_context),
tool_name: "Bash".to_string(),
tool_use_id,
command,
tool_response,
};
let preview_runs = sess.hooks().preview_post_tool_use(&request);
emit_hook_started_events(sess, turn_context, preview_runs).await;
let outcome = sess.hooks().run_post_tool_use(request).await;
emit_hook_completed_events(sess, turn_context, outcome.hook_events.clone()).await;
outcome
}
pub(crate) async fn run_user_prompt_submit_hooks(
sess: &Arc<Session>,
turn_context: &Arc<TurnContext>,
prompt: String,
) -> HookRuntimeOutcome {
let request = UserPromptSubmitRequest {
session_id: sess.conversation_id,
turn_id: turn_context.sub_id.clone(),
cwd: turn_context.cwd.clone(),
transcript_path: sess.hook_transcript_path().await,
model: turn_context.model_info.slug.clone(),
permission_mode: hook_permission_mode(turn_context),
prompt,
};
let preview_runs = sess.hooks().preview_user_prompt_submit(&request);
run_context_injecting_hook(
sess,
turn_context,
preview_runs,
sess.hooks().run_user_prompt_submit(request),
)
.await
}
pub(crate) async fn inspect_pending_input(
sess: &Arc<Session>,
turn_context: &Arc<TurnContext>,
pending_input_item: ResponseInputItem,
) -> PendingInputHookDisposition {
let response_item = ResponseItem::from(pending_input_item);
if let Some(TurnItem::UserMessage(user_message)) = parse_turn_item(&response_item) {
let user_prompt_submit_outcome =
run_user_prompt_submit_hooks(sess, turn_context, user_message.message()).await;
if user_prompt_submit_outcome.should_stop {
PendingInputHookDisposition::Blocked {
additional_contexts: user_prompt_submit_outcome.additional_contexts,
}
} else {
PendingInputHookDisposition::Accepted(Box::new(PendingInputRecord::UserMessage {
content: user_message.content,
response_item,
additional_contexts: user_prompt_submit_outcome.additional_contexts,
}))
}
} else {
PendingInputHookDisposition::Accepted(Box::new(PendingInputRecord::ConversationItem {
response_item,
}))
}
}
pub(crate) async fn record_pending_input(
sess: &Arc<Session>,
turn_context: &Arc<TurnContext>,
pending_input: PendingInputRecord,
) {
match pending_input {
PendingInputRecord::UserMessage {
content,
response_item,
additional_contexts,
} => {
sess.record_user_prompt_and_emit_turn_item(
turn_context.as_ref(),
content.as_slice(),
response_item,
)
.await;
record_additional_contexts(sess, turn_context, additional_contexts).await;
}
PendingInputRecord::ConversationItem { response_item } => {
sess.record_conversation_items(turn_context, std::slice::from_ref(&response_item))
.await;
}
}
}
async fn run_context_injecting_hook<Fut, Outcome>(
sess: &Arc<Session>,
turn_context: &Arc<TurnContext>,
preview_runs: Vec<HookRunSummary>,
outcome_future: Fut,
) -> HookRuntimeOutcome
where
Fut: Future<Output = Outcome>,
Outcome: Into<ContextInjectingHookOutcome>,
{
emit_hook_started_events(sess, turn_context, preview_runs).await;
let outcome = outcome_future.await.into();
emit_hook_completed_events(sess, turn_context, outcome.hook_events).await;
outcome.outcome
}
impl HookRuntimeOutcome {
async fn record_additional_contexts(
self,
sess: &Arc<Session>,
turn_context: &Arc<TurnContext>,
) -> bool {
record_additional_contexts(sess, turn_context, self.additional_contexts).await;
self.should_stop
}
}
pub(crate) async fn record_additional_contexts(
sess: &Arc<Session>,
turn_context: &Arc<TurnContext>,
additional_contexts: Vec<String>,
) {
let developer_messages = additional_context_messages(additional_contexts);
if developer_messages.is_empty() {
return;
}
sess.record_conversation_items(turn_context, developer_messages.as_slice())
.await;
}
fn additional_context_messages(additional_contexts: Vec<String>) -> Vec<ResponseItem> {
additional_contexts
.into_iter()
.map(|additional_context| DeveloperInstructions::new(additional_context).into())
.collect()
}
async fn emit_hook_started_events(
sess: &Arc<Session>,
turn_context: &Arc<TurnContext>,
preview_runs: Vec<HookRunSummary>,
) {
for run in preview_runs {
sess.send_event(
turn_context,
EventMsg::HookStarted(HookStartedEvent {
turn_id: Some(turn_context.sub_id.clone()),
run,
}),
)
.await;
}
}
pub(crate) async fn emit_hook_completed_events(
sess: &Arc<Session>,
turn_context: &Arc<TurnContext>,
completed_events: Vec<HookCompletedEvent>,
) {
for completed in completed_events {
track_hook_completed_analytics(sess, turn_context, &completed);
sess.send_event(turn_context, EventMsg::HookCompleted(completed))
.await;
}
}
fn track_hook_completed_analytics(
sess: &Arc<Session>,
turn_context: &Arc<TurnContext>,
completed: &HookCompletedEvent,
) {
let (tracking, hook) =
hook_run_analytics_payload(sess.conversation_id.to_string(), turn_context, completed);
sess.services
.analytics_events_client
.track_hook_run(tracking, hook);
}
fn hook_run_analytics_payload(
thread_id: String,
turn_context: &TurnContext,
completed: &HookCompletedEvent,
) -> (codex_analytics::TrackEventsContext, HookRunFact) {
(
build_track_events_context(
turn_context.model_info.slug.clone(),
thread_id,
completed
.turn_id
.clone()
.unwrap_or_else(|| turn_context.sub_id.clone()),
),
HookRunFact {
event_name: completed.run.event_name,
hook_source: completed.run.source,
status: completed.run.status,
},
)
}
fn hook_permission_mode(turn_context: &TurnContext) -> String {
match turn_context.approval_policy.value() {
AskForApproval::Never => "bypassPermissions",
AskForApproval::UnlessTrusted
| AskForApproval::OnFailure
| AskForApproval::OnRequest
| AskForApproval::Granular(_) => "default",
}
.to_string()
}
#[cfg(test)]
mod tests {
use codex_protocol::models::ContentItem;
use codex_protocol::protocol::HookEventName;
use codex_protocol::protocol::HookExecutionMode;
use codex_protocol::protocol::HookHandlerType;
use codex_protocol::protocol::HookRunStatus;
use codex_protocol::protocol::HookScope;
use codex_protocol::protocol::HookSource;
use pretty_assertions::assert_eq;
use super::additional_context_messages;
use super::hook_run_analytics_payload;
use crate::codex::make_session_and_context;
use codex_protocol::protocol::HookCompletedEvent;
use codex_protocol::protocol::HookRunSummary;
use codex_utils_absolute_path::test_support::PathBufExt;
use codex_utils_absolute_path::test_support::test_path_buf;
#[test]
fn additional_context_messages_stay_separate_and_ordered() {
let messages = additional_context_messages(vec![
"first tide note".to_string(),
"second tide note".to_string(),
]);
assert_eq!(messages.len(), 2);
assert_eq!(
messages
.iter()
.map(|message| match message {
codex_protocol::models::ResponseItem::Message { role, content, .. } => {
let text = content
.iter()
.map(|item| match item {
ContentItem::InputText { text } => text.as_str(),
ContentItem::InputImage { .. } | ContentItem::OutputText { .. } => {
panic!("expected input text content, got {item:?}")
}
})
.collect::<String>();
(role.as_str(), text)
}
other => panic!("expected developer message, got {other:?}"),
})
.collect::<Vec<_>>(),
vec![
("developer", "first tide note".to_string()),
("developer", "second tide note".to_string()),
],
);
}
#[tokio::test]
async fn hook_run_analytics_payload_uses_completed_turn_id() {
let (_session, turn_context) = make_session_and_context().await;
let completed = HookCompletedEvent {
turn_id: Some("turn-from-hook".to_string()),
run: sample_hook_run(HookRunStatus::Blocked, HookSource::Project),
};
let (tracking, hook) =
hook_run_analytics_payload("thread-123".to_string(), &turn_context, &completed);
assert_eq!(tracking.thread_id, "thread-123");
assert_eq!(tracking.turn_id, "turn-from-hook");
assert_eq!(tracking.model_slug, turn_context.model_info.slug);
assert_eq!(hook.event_name, HookEventName::Stop);
assert_eq!(hook.hook_source, HookSource::Project);
assert_eq!(hook.status, HookRunStatus::Blocked);
}
#[tokio::test]
async fn hook_run_analytics_payload_falls_back_to_turn_context_id() {
let (_session, turn_context) = make_session_and_context().await;
let completed = HookCompletedEvent {
turn_id: None,
run: sample_hook_run(HookRunStatus::Failed, HookSource::Unknown),
};
let (tracking, hook) =
hook_run_analytics_payload("thread-123".to_string(), &turn_context, &completed);
assert_eq!(tracking.turn_id, turn_context.sub_id);
assert_eq!(hook.hook_source, HookSource::Unknown);
assert_eq!(hook.status, HookRunStatus::Failed);
}
fn sample_hook_run(status: HookRunStatus, source: HookSource) -> HookRunSummary {
HookRunSummary {
id: "stop:0:/tmp/hooks.json".to_string(),
event_name: HookEventName::Stop,
handler_type: HookHandlerType::Command,
execution_mode: HookExecutionMode::Sync,
scope: HookScope::Turn,
source_path: test_path_buf("/tmp/hooks.json").abs(),
source,
display_order: 0,
status,
status_message: None,
started_at: 10,
completed_at: Some(37),
duration_ms: Some(27),
entries: Vec::new(),
}
}
}