[hooks] add non-streaming (non-stdin style) shell-only PostToolUse support (#15531)

CHAINED PR - note that base is eternal/hooks-pretooluse-bash, not main
-- so the following PR should be first

Matching post-tool hook to the pre-tool functionality here:
https://github.com/openai/codex/pull/15211

So, PreToolUse calls for plain shell calls, allows blocking. This
PostToolUse call runs after the command executed

example run:

```
› as a test, run in parallel the following commands:
  - echo 'one'
  - echo '[block-pre-tool-use]'
  - echo '[block-post-tool-use]'


⚠ MCP startup incomplete (failed: notion, linear)

• Cruising through those three commands in parallel now, and I’ll share the exact outputs right after
  they land.

• Running PreToolUse hook: checking the observatory runes

• Running PreToolUse hook: checking the observatory runes

• Running PreToolUse hook: checking the observatory runes

PreToolUse hook (blocked)
  warning: wizard-tower PreToolUse demo blocked a Bash command on purpose.
  feedback: PreToolUse demo blocked the command. Remove [block-pre-tool-use] to continue.

PreToolUse hook (completed)
  warning: wizard-tower PreToolUse demo inspected Bash: echo 'one'

PreToolUse hook (completed)
  warning: wizard-tower PreToolUse demo inspected Bash: echo '[block-post-tool-use]'

• Ran echo '[block-post-tool-use]'
  └ [block-post-tool-use]

• Ran echo 'one'
  └ one

• Running PostToolUse hook: reading the observatory aftermath

• Running PostToolUse hook: reading the observatory aftermath

PostToolUse hook (blocked)
  warning: wizard-tower PostToolUse demo blocked the Bash follow-through on purpose.
  feedback: PostToolUse demo blocked the result after execution. Remove [block-post-tool-use] to
continue.

PostToolUse hook (completed)
  warning: wizard-tower PostToolUse demo inspected Bash: echo 'one'
  hook context: PostToolUse demo saw the Bash result. Response preview: one

──────────────────────────────────────────────────────────────────────────────────────────────────────

• Ran them in parallel. Little wave report:

  - echo 'one' completed and returned one.
  - echo '[block-pre-tool-use]' was blocked before execution by the PreToolUse hook.
  - echo '[block-post-tool-use]' ran, but its result was blocked afterward by the PostToolUse hook.

  The hook messages were:

  - PreToolUse demo blocked the command. Remove [block-pre-tool-use] to continue.
  - PostToolUse demo blocked the result after execution. Remove [block-post-tool-use] to continue.


› for the blocked messages (correctly blocked in this test) did the tool output land back into
  context?


• Yeah, from this run:

  - PreToolUse block: the command itself did not execute, so no command output landed in context. Only
    the block message did.
  - PostToolUse block: the command did execute, but the actual stdout did not come back into context.
    Only the post-tool block message did.

  So the blocked notifications landed in context, but the blocked command payloads themselves did not.
  Nice clean guardrail, aloha.
```
This commit is contained in:
Andrei Eternal
2026-03-25 19:18:03 -07:00
committed by GitHub
parent 78799c1bcf
commit c4d9887f9a
43 changed files with 2078 additions and 165 deletions

View File

@@ -237,6 +237,78 @@ elif mode == "exit_2":
Ok(())
}
fn write_post_tool_use_hook(
home: &Path,
matcher: Option<&str>,
mode: &str,
reason: &str,
) -> Result<()> {
let script_path = home.join("post_tool_use_hook.py");
let log_path = home.join("post_tool_use_hook_log.jsonl");
let mode_json = serde_json::to_string(mode).context("serialize post tool use mode")?;
let reason_json = serde_json::to_string(reason).context("serialize post tool use reason")?;
let script = format!(
r#"import json
from pathlib import Path
import sys
log_path = Path(r"{log_path}")
mode = {mode_json}
reason = {reason_json}
payload = json.load(sys.stdin)
with log_path.open("a", encoding="utf-8") as handle:
handle.write(json.dumps(payload) + "\n")
if mode == "context":
print(json.dumps({{
"hookSpecificOutput": {{
"hookEventName": "PostToolUse",
"additionalContext": reason
}}
}}))
elif mode == "decision_block":
print(json.dumps({{
"decision": "block",
"reason": reason
}}))
elif mode == "continue_false":
print(json.dumps({{
"continue": False,
"stopReason": reason
}}))
elif mode == "exit_2":
sys.stderr.write(reason + "\n")
raise SystemExit(2)
"#,
log_path = log_path.display(),
mode_json = mode_json,
reason_json = reason_json,
);
let mut group = serde_json::json!({
"hooks": [{
"type": "command",
"command": format!("python3 {}", script_path.display()),
"statusMessage": "running post tool use hook",
}]
});
if let Some(matcher) = matcher {
group["matcher"] = Value::String(matcher.to_string());
}
let hooks = serde_json::json!({
"hooks": {
"PostToolUse": [group]
}
});
fs::write(&script_path, script).context("write post tool use hook script")?;
fs::write(home.join("hooks.json"), hooks.to_string()).context("write hooks.json")?;
Ok(())
}
fn write_session_start_hook_recording_transcript(home: &Path) -> Result<()> {
let script_path = home.join("session_start_hook.py");
let log_path = home.join("session_start_hook_log.jsonl");
@@ -325,6 +397,15 @@ fn read_pre_tool_use_hook_inputs(home: &Path) -> Result<Vec<serde_json::Value>>
.collect()
}
fn read_post_tool_use_hook_inputs(home: &Path) -> Result<Vec<serde_json::Value>> {
fs::read_to_string(home.join("post_tool_use_hook_log.jsonl"))
.context("read post tool use hook log")?
.lines()
.filter(|line| !line.trim().is_empty())
.map(|line| serde_json::from_str(line).context("parse post tool use hook log line"))
.collect()
}
fn read_session_start_hook_inputs(home: &Path) -> Result<Vec<serde_json::Value>> {
fs::read_to_string(home.join("session_start_hook_log.jsonl"))
.context("read session start hook log")?
@@ -986,7 +1067,7 @@ async fn pre_tool_use_blocks_shell_command_before_execution() -> Result<()> {
.and_then(Value::as_str)
.expect("shell command output string");
assert!(
output.contains("Bash command blocked by hook: blocked by pre hook"),
output.contains("Command blocked by PreToolUse hook: blocked by pre hook"),
"blocked tool output should surface the hook reason",
);
assert!(
@@ -1088,7 +1169,7 @@ async fn pre_tool_use_blocks_local_shell_before_execution() -> Result<()> {
.and_then(Value::as_str)
.expect("local shell output string");
assert!(
output.contains("Bash command blocked by hook: blocked local shell"),
output.contains("Command blocked by PreToolUse hook: blocked local shell"),
"blocked local shell output should surface the hook reason",
);
assert!(
@@ -1183,7 +1264,7 @@ async fn pre_tool_use_blocks_exec_command_before_execution() -> Result<()> {
.and_then(Value::as_str)
.expect("exec command output string");
assert!(
output.contains("Bash command blocked by hook: blocked exec command"),
output.contains("Command blocked by PreToolUse hook: blocked exec command"),
"blocked exec command output should surface the hook reason",
);
assert!(
@@ -1275,3 +1356,458 @@ async fn pre_tool_use_does_not_fire_for_non_shell_tools() -> Result<()> {
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn post_tool_use_records_additional_context_for_shell_command() -> Result<()> {
skip_if_no_network!(Ok(()));
let server = start_mock_server().await;
let call_id = "posttooluse-shell-command";
let command = "printf post-tool-output".to_string();
let args = serde_json::json!({ "command": command });
let responses = mount_sse_sequence(
&server,
vec![
sse(vec![
ev_response_created("resp-1"),
core_test_support::responses::ev_function_call(
call_id,
"shell_command",
&serde_json::to_string(&args)?,
),
ev_completed("resp-1"),
]),
sse(vec![
ev_response_created("resp-2"),
ev_assistant_message("msg-1", "post hook context observed"),
ev_completed("resp-2"),
]),
],
)
.await;
let post_context = "Remember the bash post-tool note.";
let mut builder = test_codex()
.with_pre_build_hook(|home| {
if let Err(error) =
write_post_tool_use_hook(home, Some("^Bash$"), "context", post_context)
{
panic!("failed to write post tool use hook test fixture: {error}");
}
})
.with_config(|config| {
config
.features
.enable(Feature::CodexHooks)
.expect("test config should allow feature update");
});
let test = builder.build(&server).await?;
test.submit_turn("run the shell command with post hook")
.await?;
let requests = responses.requests();
assert_eq!(requests.len(), 2);
assert!(
requests[1]
.message_input_texts("developer")
.contains(&post_context.to_string()),
"follow-up request should include post tool use additional context",
);
let output_item = requests[1].function_call_output(call_id);
let output = output_item
.get("output")
.and_then(Value::as_str)
.expect("shell command output string");
assert!(
output.contains("post-tool-output"),
"shell command output should still reach the model",
);
let hook_inputs = read_post_tool_use_hook_inputs(test.codex_home_path())?;
assert_eq!(hook_inputs.len(), 1);
assert_eq!(hook_inputs[0]["hook_event_name"], "PostToolUse");
assert_eq!(hook_inputs[0]["tool_name"], "Bash");
assert_eq!(hook_inputs[0]["tool_use_id"], call_id);
assert_eq!(hook_inputs[0]["tool_input"]["command"], command);
assert_eq!(
hook_inputs[0]["tool_response"],
Value::String("post-tool-output".to_string())
);
let transcript_path = hook_inputs[0]["transcript_path"]
.as_str()
.expect("post tool use hook transcript_path");
assert!(
!transcript_path.is_empty(),
"post tool use hook should receive a non-empty transcript_path",
);
assert!(
Path::new(transcript_path).exists(),
"post tool use hook transcript_path should be materialized on disk",
);
assert!(
hook_inputs[0]["turn_id"]
.as_str()
.is_some_and(|turn_id| !turn_id.is_empty())
);
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn post_tool_use_block_decision_replaces_shell_command_output_with_reason() -> Result<()> {
skip_if_no_network!(Ok(()));
let server = start_mock_server().await;
let call_id = "posttooluse-shell-command-block";
let command = "printf blocked-output".to_string();
let args = serde_json::json!({ "command": command });
let responses = mount_sse_sequence(
&server,
vec![
sse(vec![
ev_response_created("resp-1"),
core_test_support::responses::ev_function_call(
call_id,
"shell_command",
&serde_json::to_string(&args)?,
),
ev_completed("resp-1"),
]),
sse(vec![
ev_response_created("resp-2"),
ev_assistant_message("msg-1", "post hook feedback observed"),
ev_completed("resp-2"),
]),
],
)
.await;
let reason = "bash output looked sketchy";
let mut builder = test_codex()
.with_pre_build_hook(|home| {
if let Err(error) =
write_post_tool_use_hook(home, Some("^Bash$"), "decision_block", reason)
{
panic!("failed to write post tool use hook test fixture: {error}");
}
})
.with_config(|config| {
config
.features
.enable(Feature::CodexHooks)
.expect("test config should allow feature update");
});
let test = builder.build(&server).await?;
test.submit_turn("run the shell command with blocking post hook")
.await?;
let requests = responses.requests();
assert_eq!(requests.len(), 2);
let output_item = requests[1].function_call_output(call_id);
let output = output_item
.get("output")
.and_then(Value::as_str)
.expect("shell command output string");
assert_eq!(output, reason);
let hook_inputs = read_post_tool_use_hook_inputs(test.codex_home_path())?;
assert_eq!(hook_inputs.len(), 1);
assert_eq!(
hook_inputs[0]["tool_response"],
Value::String("blocked-output".to_string())
);
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn post_tool_use_continue_false_replaces_shell_command_output_with_stop_reason() -> Result<()>
{
skip_if_no_network!(Ok(()));
let server = start_mock_server().await;
let call_id = "posttooluse-shell-command-stop";
let command = "printf stop-output".to_string();
let args = serde_json::json!({ "command": command });
let responses = mount_sse_sequence(
&server,
vec![
sse(vec![
ev_response_created("resp-1"),
core_test_support::responses::ev_function_call(
call_id,
"shell_command",
&serde_json::to_string(&args)?,
),
ev_completed("resp-1"),
]),
sse(vec![
ev_response_created("resp-2"),
ev_assistant_message("msg-1", "post hook stop observed"),
ev_completed("resp-2"),
]),
],
)
.await;
let stop_reason = "Execution halted by post-tool hook";
let mut builder = test_codex()
.with_pre_build_hook(|home| {
if let Err(error) =
write_post_tool_use_hook(home, Some("^Bash$"), "continue_false", stop_reason)
{
panic!("failed to write post tool use hook test fixture: {error}");
}
})
.with_config(|config| {
config
.features
.enable(Feature::CodexHooks)
.expect("test config should allow feature update");
});
let test = builder.build(&server).await?;
test.submit_turn("run the shell command with stop-style post hook")
.await?;
let requests = responses.requests();
assert_eq!(requests.len(), 2);
let output_item = requests[1].function_call_output(call_id);
let output = output_item
.get("output")
.and_then(Value::as_str)
.expect("shell command output string");
assert_eq!(output, stop_reason);
let hook_inputs = read_post_tool_use_hook_inputs(test.codex_home_path())?;
assert_eq!(hook_inputs.len(), 1);
assert_eq!(
hook_inputs[0]["tool_response"],
Value::String("stop-output".to_string())
);
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn post_tool_use_records_additional_context_for_local_shell() -> Result<()> {
skip_if_no_network!(Ok(()));
let server = start_mock_server().await;
let call_id = "posttooluse-local-shell";
let command = vec![
"/bin/sh".to_string(),
"-c".to_string(),
"printf local-post-tool-output".to_string(),
];
let responses = mount_sse_sequence(
&server,
vec![
sse(vec![
ev_response_created("resp-1"),
core_test_support::responses::ev_local_shell_call(
call_id,
"completed",
command.iter().map(String::as_str).collect(),
),
ev_completed("resp-1"),
]),
sse(vec![
ev_response_created("resp-2"),
ev_assistant_message("msg-1", "local shell post hook context observed"),
ev_completed("resp-2"),
]),
],
)
.await;
let post_context = "Remember the local shell post-tool note.";
let mut builder = test_codex()
.with_pre_build_hook(|home| {
if let Err(error) =
write_post_tool_use_hook(home, Some("^Bash$"), "context", post_context)
{
panic!("failed to write post tool use hook test fixture: {error}");
}
})
.with_config(|config| {
config
.features
.enable(Feature::CodexHooks)
.expect("test config should allow feature update");
});
let test = builder.build(&server).await?;
test.submit_turn("run the local shell command with post hook")
.await?;
let requests = responses.requests();
assert_eq!(requests.len(), 2);
assert!(
requests[1]
.message_input_texts("developer")
.contains(&post_context.to_string()),
"follow-up request should include local shell post tool use additional context",
);
let hook_inputs = read_post_tool_use_hook_inputs(test.codex_home_path())?;
assert_eq!(hook_inputs.len(), 1);
assert_eq!(
hook_inputs[0]["tool_input"]["command"],
codex_shell_command::parse_command::shlex_join(&command),
);
assert_eq!(
hook_inputs[0]["tool_response"],
Value::String("local-post-tool-output".to_string()),
);
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn post_tool_use_exit_two_replaces_one_shot_exec_command_output_with_feedback() -> Result<()>
{
skip_if_no_network!(Ok(()));
let server = start_mock_server().await;
let call_id = "posttooluse-exec-command";
let command = "printf post-hook-output".to_string();
let args = serde_json::json!({ "cmd": command, "tty": false });
let responses = mount_sse_sequence(
&server,
vec![
sse(vec![
ev_response_created("resp-1"),
core_test_support::responses::ev_function_call(
call_id,
"exec_command",
&serde_json::to_string(&args)?,
),
ev_completed("resp-1"),
]),
sse(vec![
ev_response_created("resp-2"),
ev_assistant_message("msg-1", "post hook blocked the exec result"),
ev_completed("resp-2"),
]),
],
)
.await;
let mut builder = test_codex()
.with_pre_build_hook(|home| {
if let Err(error) =
write_post_tool_use_hook(home, Some("^Bash$"), "exit_2", "blocked by post hook")
{
panic!("failed to write post tool use hook test fixture: {error}");
}
})
.with_config(|config| {
config.use_experimental_unified_exec_tool = true;
config
.features
.enable(Feature::CodexHooks)
.expect("test config should allow feature update");
config
.features
.enable(Feature::UnifiedExec)
.expect("test config should allow feature update");
});
let test = builder.build(&server).await?;
test.submit_turn("run the exec command with post hook")
.await?;
let requests = responses.requests();
assert_eq!(requests.len(), 2);
let output_item = requests[1].function_call_output(call_id);
let output = output_item
.get("output")
.and_then(Value::as_str)
.expect("exec command output string");
assert_eq!(output, "blocked by post hook");
let hook_inputs = read_post_tool_use_hook_inputs(test.codex_home_path())?;
assert_eq!(hook_inputs.len(), 1);
assert_eq!(hook_inputs[0]["tool_use_id"], call_id);
assert_eq!(hook_inputs[0]["tool_input"]["command"], command);
assert_eq!(
hook_inputs[0]["tool_response"],
Value::String("post-hook-output".to_string())
);
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn post_tool_use_does_not_fire_for_non_shell_tools() -> Result<()> {
skip_if_no_network!(Ok(()));
let server = start_mock_server().await;
let call_id = "posttooluse-update-plan";
let args = serde_json::json!({
"plan": [{
"step": "watch the tide",
"status": "pending",
}]
});
let responses = mount_sse_sequence(
&server,
vec![
sse(vec![
ev_response_created("resp-1"),
core_test_support::responses::ev_function_call(
call_id,
"update_plan",
&serde_json::to_string(&args)?,
),
ev_completed("resp-1"),
]),
sse(vec![
ev_response_created("resp-2"),
ev_assistant_message("msg-1", "plan updated"),
ev_completed("resp-2"),
]),
],
)
.await;
let mut builder = test_codex()
.with_pre_build_hook(|home| {
if let Err(error) =
write_post_tool_use_hook(home, None, "decision_block", "should not fire")
{
panic!("failed to write post tool use hook test fixture: {error}");
}
})
.with_config(|config| {
config
.features
.enable(Feature::CodexHooks)
.expect("test config should allow feature update");
});
let test = builder.build(&server).await?;
test.submit_turn("update the plan").await?;
let requests = responses.requests();
assert_eq!(requests.len(), 2);
let output_item = requests[1].function_call_output(call_id);
let output = output_item
.get("output")
.and_then(Value::as_str)
.expect("update plan output string");
assert!(
!output.contains("should not fire"),
"non-shell tool output should not be affected by PostToolUse",
);
let hook_log_path = test.codex_home_path().join("post_tool_use_hook_log.jsonl");
assert!(
!hook_log_path.exists(),
"non-shell tools should not trigger post tool use hooks",
);
Ok(())
}