mirror of
https://github.com/openai/codex.git
synced 2026-04-26 23:55:25 +00:00
Use realtime transcript for handoff context (#14132)
- collect input/output transcript deltas into active handoff transcript state - attach and clear that transcript on each handoff, and regenerate schema/tests
This commit is contained in:
@@ -970,12 +970,15 @@ async fn conversation_mirrors_assistant_message_text_to_realtime_handoff() -> Re
|
||||
"type": "session.updated",
|
||||
"session": { "id": "sess_1", "instructions": "backend prompt" }
|
||||
}),
|
||||
json!({
|
||||
"type": "conversation.input_transcript.delta",
|
||||
"delta": "delegate hello"
|
||||
}),
|
||||
json!({
|
||||
"type": "conversation.handoff.requested",
|
||||
"handoff_id": "handoff_1",
|
||||
"item_id": "item_1",
|
||||
"input_transcript": "delegate hello",
|
||||
"messages": [{ "role": "user", "text": "delegate hello" }]
|
||||
"input_transcript": "delegate hello"
|
||||
}),
|
||||
],
|
||||
vec![],
|
||||
@@ -1089,12 +1092,15 @@ async fn conversation_handoff_persists_across_item_done_until_turn_complete() ->
|
||||
"type": "session.updated",
|
||||
"session": { "id": "sess_item_done", "instructions": "backend prompt" }
|
||||
}),
|
||||
json!({
|
||||
"type": "conversation.input_transcript.delta",
|
||||
"delta": "delegate now"
|
||||
}),
|
||||
json!({
|
||||
"type": "conversation.handoff.requested",
|
||||
"handoff_id": "handoff_item_done",
|
||||
"item_id": "item_item_done",
|
||||
"input_transcript": "delegate now",
|
||||
"messages": [{ "role": "user", "text": "delegate now" }]
|
||||
"input_transcript": "delegate now"
|
||||
}),
|
||||
],
|
||||
vec![json!({
|
||||
@@ -1229,12 +1235,15 @@ async fn inbound_handoff_request_starts_turn() -> Result<()> {
|
||||
"type": "session.updated",
|
||||
"session": { "id": "sess_inbound", "instructions": "backend prompt" }
|
||||
}),
|
||||
json!({
|
||||
"type": "conversation.input_transcript.delta",
|
||||
"delta": "text from realtime"
|
||||
}),
|
||||
json!({
|
||||
"type": "conversation.handoff.requested",
|
||||
"handoff_id": "handoff_inbound",
|
||||
"item_id": "item_inbound",
|
||||
"input_transcript": "text from realtime",
|
||||
"messages": [{ "role": "user", "text": "text from realtime" }]
|
||||
"input_transcript": "text from realtime"
|
||||
}),
|
||||
]]])
|
||||
.await;
|
||||
@@ -1293,7 +1302,7 @@ async fn inbound_handoff_request_starts_turn() -> Result<()> {
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn inbound_handoff_request_uses_all_messages() -> Result<()> {
|
||||
async fn inbound_handoff_request_uses_active_transcript() -> Result<()> {
|
||||
skip_if_no_network!(Ok(()));
|
||||
|
||||
let api_server = start_mock_server().await;
|
||||
@@ -1312,16 +1321,23 @@ async fn inbound_handoff_request_uses_all_messages() -> Result<()> {
|
||||
"type": "session.updated",
|
||||
"session": { "id": "sess_inbound_multi", "instructions": "backend prompt" }
|
||||
}),
|
||||
json!({
|
||||
"type": "conversation.output_transcript.delta",
|
||||
"delta": "assistant context"
|
||||
}),
|
||||
json!({
|
||||
"type": "conversation.input_transcript.delta",
|
||||
"delta": "delegated query"
|
||||
}),
|
||||
json!({
|
||||
"type": "conversation.output_transcript.delta",
|
||||
"delta": "assist confirm"
|
||||
}),
|
||||
json!({
|
||||
"type": "conversation.handoff.requested",
|
||||
"handoff_id": "handoff_inbound_multi",
|
||||
"item_id": "item_inbound_multi",
|
||||
"input_transcript": "ignored",
|
||||
"messages": [
|
||||
{ "role": "assistant", "text": "assistant context" },
|
||||
{ "role": "user", "text": "delegated query" },
|
||||
{ "role": "assistant", "text": "assist confirm" },
|
||||
]
|
||||
"input_transcript": "ignored"
|
||||
}),
|
||||
]]])
|
||||
.await;
|
||||
@@ -1363,6 +1379,131 @@ async fn inbound_handoff_request_uses_all_messages() -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn inbound_handoff_request_clears_active_transcript_after_each_handoff() -> Result<()> {
|
||||
skip_if_no_network!(Ok(()));
|
||||
|
||||
let api_server = start_mock_server().await;
|
||||
let response_mock = responses::mount_sse_sequence(
|
||||
&api_server,
|
||||
vec![
|
||||
responses::sse(vec![
|
||||
responses::ev_response_created("resp-1"),
|
||||
responses::ev_assistant_message("msg-1", "first ok"),
|
||||
responses::ev_completed("resp-1"),
|
||||
]),
|
||||
responses::sse(vec![
|
||||
responses::ev_response_created("resp-2"),
|
||||
responses::ev_assistant_message("msg-2", "second ok"),
|
||||
responses::ev_completed("resp-2"),
|
||||
]),
|
||||
],
|
||||
)
|
||||
.await;
|
||||
|
||||
let realtime_server = start_websocket_server(vec![vec![
|
||||
vec![
|
||||
json!({
|
||||
"type": "session.updated",
|
||||
"session": { "id": "sess_inbound_clear", "instructions": "backend prompt" }
|
||||
}),
|
||||
json!({
|
||||
"type": "conversation.input_transcript.delta",
|
||||
"delta": "first question"
|
||||
}),
|
||||
json!({
|
||||
"type": "conversation.handoff.requested",
|
||||
"handoff_id": "handoff_inbound_clear_1",
|
||||
"item_id": "item_inbound_clear_1",
|
||||
"input_transcript": "first question"
|
||||
}),
|
||||
],
|
||||
vec![],
|
||||
vec![
|
||||
json!({
|
||||
"type": "conversation.input_transcript.delta",
|
||||
"delta": "second question"
|
||||
}),
|
||||
json!({
|
||||
"type": "conversation.handoff.requested",
|
||||
"handoff_id": "handoff_inbound_clear_2",
|
||||
"item_id": "item_inbound_clear_2",
|
||||
"input_transcript": "second question"
|
||||
}),
|
||||
],
|
||||
]])
|
||||
.await;
|
||||
|
||||
let mut builder = test_codex().with_config({
|
||||
let realtime_base_url = realtime_server.uri().to_string();
|
||||
move |config| {
|
||||
config.experimental_realtime_ws_base_url = Some(realtime_base_url);
|
||||
}
|
||||
});
|
||||
let test = builder.build(&api_server).await?;
|
||||
|
||||
test.codex
|
||||
.submit(Op::RealtimeConversationStart(ConversationStartParams {
|
||||
prompt: "backend prompt".to_string(),
|
||||
session_id: None,
|
||||
}))
|
||||
.await?;
|
||||
|
||||
let _ = wait_for_event_match(&test.codex, |msg| match msg {
|
||||
EventMsg::RealtimeConversationRealtime(RealtimeConversationRealtimeEvent {
|
||||
payload: RealtimeEvent::SessionUpdated { session_id, .. },
|
||||
}) => Some(session_id.clone()),
|
||||
_ => None,
|
||||
})
|
||||
.await;
|
||||
|
||||
wait_for_event(&test.codex, |event| {
|
||||
matches!(event, EventMsg::TurnComplete(_))
|
||||
})
|
||||
.await;
|
||||
|
||||
test.codex
|
||||
.submit(Op::RealtimeConversationAudio(ConversationAudioParams {
|
||||
frame: RealtimeAudioFrame {
|
||||
data: "AQID".to_string(),
|
||||
sample_rate: 24000,
|
||||
num_channels: 1,
|
||||
samples_per_channel: Some(480),
|
||||
},
|
||||
}))
|
||||
.await?;
|
||||
|
||||
wait_for_event(&test.codex, |event| {
|
||||
matches!(event, EventMsg::TurnComplete(_))
|
||||
})
|
||||
.await;
|
||||
|
||||
let requests = response_mock.requests();
|
||||
assert_eq!(requests.len(), 2);
|
||||
|
||||
let first_user_texts = requests[0].message_input_texts("user");
|
||||
assert!(
|
||||
first_user_texts
|
||||
.iter()
|
||||
.any(|text| text == "user: first question")
|
||||
);
|
||||
|
||||
let second_user_texts = requests[1].message_input_texts("user");
|
||||
assert!(
|
||||
second_user_texts
|
||||
.iter()
|
||||
.any(|text| text == "user: second question")
|
||||
);
|
||||
assert!(
|
||||
!second_user_texts
|
||||
.iter()
|
||||
.any(|text| text == "user: first question\nuser: second question")
|
||||
);
|
||||
|
||||
realtime_server.shutdown().await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn inbound_conversation_item_does_not_start_turn_and_still_forwards_audio() -> Result<()> {
|
||||
skip_if_no_network!(Ok(()));
|
||||
@@ -1473,12 +1614,15 @@ async fn delegated_turn_user_role_echo_does_not_redelegate_and_still_forwards_au
|
||||
"type": "session.updated",
|
||||
"session": { "id": "sess_echo_guard", "instructions": "backend prompt" }
|
||||
}),
|
||||
json!({
|
||||
"type": "conversation.input_transcript.delta",
|
||||
"delta": "delegate now"
|
||||
}),
|
||||
json!({
|
||||
"type": "conversation.handoff.requested",
|
||||
"handoff_id": "handoff_echo_guard",
|
||||
"item_id": "item_echo_guard",
|
||||
"input_transcript": "delegate now",
|
||||
"messages": [{"role": "user", "text": "delegate now"}]
|
||||
"input_transcript": "delegate now"
|
||||
}),
|
||||
],
|
||||
vec![
|
||||
@@ -1621,12 +1765,15 @@ async fn inbound_handoff_request_does_not_block_realtime_event_forwarding() -> R
|
||||
"type": "session.updated",
|
||||
"session": { "id": "sess_non_blocking", "instructions": "backend prompt" }
|
||||
}),
|
||||
json!({
|
||||
"type": "conversation.input_transcript.delta",
|
||||
"delta": "delegate now"
|
||||
}),
|
||||
json!({
|
||||
"type": "conversation.handoff.requested",
|
||||
"handoff_id": "handoff_non_blocking",
|
||||
"item_id": "item_non_blocking",
|
||||
"input_transcript": "delegate now",
|
||||
"messages": [{"role": "user", "text": "delegate now"}]
|
||||
"input_transcript": "delegate now"
|
||||
}),
|
||||
json!({
|
||||
"type": "conversation.output_audio.delta",
|
||||
@@ -1748,13 +1895,18 @@ async fn inbound_handoff_request_steers_active_turn() -> Result<()> {
|
||||
"type": "session.updated",
|
||||
"session": { "id": "sess_steer", "instructions": "backend prompt" }
|
||||
})],
|
||||
vec![json!({
|
||||
"type": "conversation.handoff.requested",
|
||||
"handoff_id": "handoff_steer",
|
||||
"item_id": "item_steer",
|
||||
"input_transcript": "steer via realtime",
|
||||
"messages": [{ "role": "user", "text": "steer via realtime" }]
|
||||
})],
|
||||
vec![
|
||||
json!({
|
||||
"type": "conversation.input_transcript.delta",
|
||||
"delta": "steer via realtime"
|
||||
}),
|
||||
json!({
|
||||
"type": "conversation.handoff.requested",
|
||||
"handoff_id": "handoff_steer",
|
||||
"item_id": "item_steer",
|
||||
"input_transcript": "steer via realtime"
|
||||
}),
|
||||
],
|
||||
]])
|
||||
.await;
|
||||
|
||||
@@ -1879,12 +2031,15 @@ async fn inbound_handoff_request_starts_turn_and_does_not_block_realtime_audio()
|
||||
"type": "session.updated",
|
||||
"session": { "id": "sess_handoff_request", "instructions": "backend prompt" }
|
||||
}),
|
||||
json!({
|
||||
"type": "conversation.input_transcript.delta",
|
||||
"delta": delegated_text
|
||||
}),
|
||||
json!({
|
||||
"type": "conversation.handoff.requested",
|
||||
"handoff_id": "handoff_audio",
|
||||
"item_id": "item_audio",
|
||||
"input_transcript": delegated_text,
|
||||
"messages": [{ "role": "user", "text": delegated_text }]
|
||||
"input_transcript": delegated_text
|
||||
}),
|
||||
json!({
|
||||
"type": "conversation.output_audio.delta",
|
||||
|
||||
Reference in New Issue
Block a user