codex: fix CI failure on PR #17676

Co-authored-by: Codex <noreply@openai.com>

codex-rs/app-server/tests/suite/v2/realtime_conversation.rs

@@ -550,9 +550,10 @@ async fn realtime_conversation_streams_v2_notifications() -> Result<()> {
         startup_context_request.body_json()["type"].as_str(),
         Some("session.update")
     );
-    assert_eq!(
-        startup_context_request.body_json()["session"]["audio"]["output"]["voice"],
-        "cedar"
+    assert!(
+        startup_context_request.body_json()["session"]["audio"]
+            .get("output")
+            .is_none()
     );
     let startup_context_instructions =
         startup_context_request.body_json()["session"]["instructions"]
@@ -973,7 +974,7 @@ async fn realtime_webrtc_start_emits_sdp_notification() -> Result<()> {
         Some("multipart/form-data; boundary=codex-realtime-call-boundary")
     );
     let body = String::from_utf8(request.body).context("multipart body should be utf-8")?;
-    let session = r#"{"tool_choice":"auto","type":"realtime","model":"gpt-realtime-1.5","instructions":"backend prompt\n\nstartup context","output_modalities":["audio"],"audio":{"input":{"format":{"type":"audio/pcm","rate":24000},"noise_reduction":{"type":"near_field"},"turn_detection":{"type":"server_vad","interrupt_response":true,"create_response":true}},"output":{"format":{"type":"audio/pcm","rate":24000},"voice":"marin"}},"tools":[{"type":"function","name":"background_agent","description":"Send a user request to the background agent. Use this as the default action. If the background agent is idle, this starts a new task and returns the final result to the user. If the background agent is already working on a task, this sends the request as guidance to steer that previous task. If the user asks to do something next, later, after this, or once current work finishes, call this tool so the work is actually queued instead of merely promising to do it later.","parameters":{"type":"object","properties":{"prompt":{"type":"string","description":"The user request to delegate to the background agent."}},"required":["prompt"],"additionalProperties":false}}]}"#;
+    let session = r#"{"tool_choice":"auto","type":"realtime","model":"gpt-realtime-1.5","instructions":"backend prompt\n\nstartup context","output_modalities":["text"],"audio":{"input":{"format":{"type":"audio/pcm","rate":24000},"noise_reduction":{"type":"near_field"},"turn_detection":{"type":"server_vad","interrupt_response":true,"create_response":true}}},"tools":[{"type":"function","name":"background_agent","description":"Send a user request to the background agent. Use this as the default action. Do not rephrase the user's ask or rewrite it in your own words; pass along the user's own words. If the background agent is idle, this starts a new task and returns the final result to the user. If the background agent is already working on a task, this sends the request as guidance to steer that previous task. If the user asks to do something next, later, after this, or once current work finishes, call this tool so the work is actually queued instead of merely promising to do it later.","parameters":{"type":"object","properties":{"prompt":{"type":"string","description":"The user request to delegate to the background agent."}},"required":["prompt"],"additionalProperties":false}}]}"#;
     assert_eq!(
         body,
         format!(
@@ -1996,6 +1997,7 @@ fn assert_v2_session_update(request: &Value) -> Result<()> {
             .context("v2 session.update instructions")?
             .contains("startup context")
     );
+    assert_eq!(request["session"]["output_modalities"], json!(["text"]));
     assert_eq!(
         request["session"]["tools"][0]["name"].as_str(),
         Some("background_agent")

codex-rs/codex-api/src/endpoint/realtime_websocket/methods.rs

@@ -1519,7 +1519,7 @@ mod tests {
                 first_json["session"]["type"],
                 Value::String("realtime".to_string())
             );
-            assert_eq!(first_json["session"]["output_modalities"], json!(["audio"]));
+            assert_eq!(first_json["session"]["output_modalities"], json!(["text"]));
             assert_eq!(
                 first_json["session"]["audio"]["input"]["format"],
                 json!({
@@ -1541,17 +1541,7 @@ mod tests {
                     "create_response": true,
                 })
             );
-            assert_eq!(
-                first_json["session"]["audio"]["output"]["format"],
-                json!({
-                    "type": "audio/pcm",
-                    "rate": 24_000,
-                })
-            );
-            assert_eq!(
-                first_json["session"]["audio"]["output"]["voice"],
-                Value::String("cedar".to_string())
-            );
+            assert!(first_json["session"]["audio"].get("output").is_none());
             assert_eq!(
                 first_json["session"]["tools"][0]["type"],
                 Value::String("function".to_string())

codex-rs/codex-api/src/endpoint/realtime_websocket/methods_v2.rs

@@ -14,8 +14,6 @@ use crate::endpoint::realtime_websocket::protocol::RealtimeVoice;
 use crate::endpoint::realtime_websocket::protocol::SessionAudio;
 use crate::endpoint::realtime_websocket::protocol::SessionAudioFormat;
 use crate::endpoint::realtime_websocket::protocol::SessionAudioInput;
-use crate::endpoint::realtime_websocket::protocol::SessionAudioOutput;
-use crate::endpoint::realtime_websocket::protocol::SessionAudioOutputFormat;
 use crate::endpoint::realtime_websocket::protocol::SessionFunctionTool;
 use crate::endpoint::realtime_websocket::protocol::SessionNoiseReduction;
 use crate::endpoint::realtime_websocket::protocol::SessionToolType;
@@ -25,10 +23,10 @@ use crate::endpoint::realtime_websocket::protocol::SessionUpdateSession;
 use crate::endpoint::realtime_websocket::protocol::TurnDetectionType;
 use serde_json::json;
 
-const REALTIME_V2_OUTPUT_MODALITY_AUDIO: &str = "audio";
+const REALTIME_V2_OUTPUT_MODALITY_TEXT: &str = "text";
 const REALTIME_V2_TOOL_CHOICE: &str = "auto";
 const REALTIME_V2_BACKGROUND_AGENT_TOOL_NAME: &str = "background_agent";
-const REALTIME_V2_BACKGROUND_AGENT_TOOL_DESCRIPTION: &str = "Send a user request to the background agent. Use this as the default action. If the background agent is idle, this starts a new task and returns the final result to the user. If the background agent is already working on a task, this sends the request as guidance to steer that previous task. If the user asks to do something next, later, after this, or once current work finishes, call this tool so the work is actually queued instead of merely promising to do it later.";
+const REALTIME_V2_BACKGROUND_AGENT_TOOL_DESCRIPTION: &str = "Send a user request to the background agent. Use this as the default action. Do not rephrase the user's ask or rewrite it in your own words; pass along the user's own words. If the background agent is idle, this starts a new task and returns the final result to the user. If the background agent is already working on a task, this sends the request as guidance to steer that previous task. If the user asks to do something next, later, after this, or once current work finishes, call this tool so the work is actually queued instead of merely promising to do it later.";
 
 pub(super) fn conversation_item_create_message(text: String) -> RealtimeOutboundMessage {
     RealtimeOutboundMessage::ConversationItemCreate {
@@ -59,7 +57,7 @@ pub(super) fn conversation_handoff_append_message(
 pub(super) fn session_update_session(
     instructions: String,
     session_mode: RealtimeSessionMode,
-    voice: RealtimeVoice,
+    _voice: RealtimeVoice,
 ) -> SessionUpdateSession {
     match session_mode {
         RealtimeSessionMode::Conversational => SessionUpdateSession {
@@ -67,7 +65,7 @@ pub(super) fn session_update_session(
             r#type: SessionType::Realtime,
             model: None,
             instructions: Some(instructions),
-            output_modalities: Some(vec![REALTIME_V2_OUTPUT_MODALITY_AUDIO.to_string()]),
+            output_modalities: Some(vec![REALTIME_V2_OUTPUT_MODALITY_TEXT.to_string()]),
             audio: SessionAudio {
                 input: SessionAudioInput {
                     format: SessionAudioFormat {
@@ -83,13 +81,7 @@ pub(super) fn session_update_session(
                         create_response: true,
                     }),
                 },
-                output: Some(SessionAudioOutput {
-                    format: Some(SessionAudioOutputFormat {
-                        r#type: AudioFormatType::AudioPcm,
-                        rate: REALTIME_AUDIO_SAMPLE_RATE,
-                    }),
-                    voice,
-                }),
+                output: None,
             },
             tools: Some(vec![SessionFunctionTool {
                 r#type: SessionToolType::Function,

codex-rs/core/tests/suite/realtime_conversation.rs

@@ -415,7 +415,7 @@ async fn conversation_start_defaults_to_v2_and_gpt_realtime_1_5() -> Result<()>
         json!({
             "startedVersion": RealtimeConversationVersion::V2,
             "handshakeUri": "/v1/realtime?model=gpt-realtime-1.5",
-            "voice": "marin",
+            "voice": null,
             "instructions": "backend prompt",
         })
     );

codex-rs/tui/src/chatwidget.rs

@@ -193,6 +193,10 @@ use codex_protocol::protocol::McpToolCallEndEvent;
 use codex_protocol::protocol::Op;
 use codex_protocol::protocol::PatchApplyBeginEvent;
 use codex_protocol::protocol::RateLimitSnapshot;
+use codex_protocol::protocol::RealtimeConversationClosedEvent;
+use codex_protocol::protocol::RealtimeConversationRealtimeEvent;
+use codex_protocol::protocol::RealtimeConversationStartedEvent;
+use codex_protocol::protocol::RealtimeEvent;
 use codex_protocol::protocol::ReviewRequest;
 use codex_protocol::protocol::ReviewTarget;
 use codex_protocol::protocol::SkillMetadata as ProtocolSkillMetadata;
@@ -337,6 +341,7 @@ use crate::history_cell::HistoryCell;
 use crate::history_cell::HookCell;
 use crate::history_cell::McpToolCallCell;
 use crate::history_cell::PlainHistoryCell;
+use crate::history_cell::RealtimeTranscriptRole;
 use crate::history_cell::WebSearchCell;
 use crate::key_hint;
 use crate::key_hint::KeyBinding;
@@ -6175,54 +6180,45 @@ impl ChatWidget {
             }
             ServerNotification::ThreadRealtimeStarted(notification) => {
                 if !from_replay {
-                    self.on_realtime_conversation_started(
-                        codex_protocol::protocol::RealtimeConversationStartedEvent {
-                            session_id: notification.session_id,
-                            version: notification.version,
-                        },
-                    );
+                    self.on_realtime_conversation_started(RealtimeConversationStartedEvent {
+                        session_id: notification.session_id,
+                        version: notification.version,
+                    });
                 }
             }
             ServerNotification::ThreadRealtimeItemAdded(notification) => {
                 if !from_replay {
-                    self.on_realtime_conversation_realtime(
-                        codex_protocol::protocol::RealtimeConversationRealtimeEvent {
-                            payload: codex_protocol::protocol::RealtimeEvent::ConversationItemAdded(
-                                notification.item,
-                            ),
-                        },
-                    );
+                    self.on_realtime_conversation_realtime(RealtimeConversationRealtimeEvent {
+                        payload: RealtimeEvent::ConversationItemAdded(notification.item),
+                    });
+                }
+            }
+            ServerNotification::ThreadRealtimeTranscriptUpdated(notification) => {
+                if !from_replay
+                    && let Some(role) = RealtimeTranscriptRole::from_name(&notification.role)
+                {
+                    self.on_realtime_transcript_delta(role, notification.text);
                 }
             }
             ServerNotification::ThreadRealtimeOutputAudioDelta(notification) => {
                 if !from_replay {
-                    self.on_realtime_conversation_realtime(
-                        codex_protocol::protocol::RealtimeConversationRealtimeEvent {
-                            payload: codex_protocol::protocol::RealtimeEvent::AudioOut(
-                                notification.audio.into(),
-                            ),
-                        },
-                    );
+                    self.on_realtime_conversation_realtime(RealtimeConversationRealtimeEvent {
+                        payload: RealtimeEvent::AudioOut(notification.audio.into()),
+                    });
                 }
             }
             ServerNotification::ThreadRealtimeError(notification) => {
                 if !from_replay {
-                    self.on_realtime_conversation_realtime(
-                        codex_protocol::protocol::RealtimeConversationRealtimeEvent {
-                            payload: codex_protocol::protocol::RealtimeEvent::Error(
-                                notification.message,
-                            ),
-                        },
-                    );
+                    self.on_realtime_conversation_realtime(RealtimeConversationRealtimeEvent {
+                        payload: RealtimeEvent::Error(notification.message),
+                    });
                 }
             }
             ServerNotification::ThreadRealtimeClosed(notification) => {
                 if !from_replay {
-                    self.on_realtime_conversation_closed(
-                        codex_protocol::protocol::RealtimeConversationClosedEvent {
-                            reason: notification.reason,
-                        },
-                    );
+                    self.on_realtime_conversation_closed(RealtimeConversationClosedEvent {
+                        reason: notification.reason,
+                    });
                 }
             }
             ServerNotification::ThreadRealtimeSdp(notification) => {
@@ -6245,7 +6241,6 @@ impl ChatWidget {
             | ServerNotification::FsChanged(_)
             | ServerNotification::FuzzyFileSearchSessionUpdated(_)
             | ServerNotification::FuzzyFileSearchSessionCompleted(_)
-            | ServerNotification::ThreadRealtimeTranscriptUpdated(_)
             | ServerNotification::WindowsWorldWritableWarning(_)
             | ServerNotification::WindowsSandboxSetupCompleted(_)
             | ServerNotification::AccountLoginCompleted(_) => {}

codex-rs/tui/src/chatwidget/realtime.rs

@@ -1,4 +1,6 @@
 use super::*;
+use crate::history_cell::RealtimeTranscriptCell;
+use crate::history_cell::RealtimeTranscriptRole;
 use codex_config::config_toml::RealtimeTransport;
 use codex_protocol::protocol::ConversationStartParams;
 use codex_protocol::protocol::ConversationStartTransport;
@@ -315,8 +317,6 @@ impl ChatWidget {
                 RealtimeEvent::AudioOut(_)
                     | RealtimeEvent::InputAudioSpeechStarted(_)
                     | RealtimeEvent::ResponseCreated(_)
-                    | RealtimeEvent::ResponseCancelled(_)
-                    | RealtimeEvent::ResponseDone(_)
             )
         {
             return;
@@ -326,12 +326,19 @@ impl ChatWidget {
                 self.realtime_conversation.session_id = Some(session_id);
             }
             RealtimeEvent::InputAudioSpeechStarted(_) => self.interrupt_realtime_audio_playback(),
-            RealtimeEvent::InputTranscriptDelta(_) => {}
-            RealtimeEvent::OutputTranscriptDelta(_) => {}
+            RealtimeEvent::InputTranscriptDelta(delta) => {
+                self.on_realtime_transcript_delta(RealtimeTranscriptRole::User, delta.delta);
+            }
+            RealtimeEvent::OutputTranscriptDelta(delta) => {
+                self.on_realtime_transcript_delta(RealtimeTranscriptRole::Assistant, delta.delta);
+            }
             RealtimeEvent::AudioOut(frame) => self.enqueue_realtime_audio_out(&frame),
             RealtimeEvent::ResponseCreated(_) => {}
-            RealtimeEvent::ResponseCancelled(_) => self.interrupt_realtime_audio_playback(),
-            RealtimeEvent::ResponseDone(_) => {}
+            RealtimeEvent::ResponseCancelled(_) => {
+                self.flush_active_realtime_transcript();
+                self.interrupt_realtime_audio_playback();
+            }
+            RealtimeEvent::ResponseDone(_) => self.flush_active_realtime_transcript(),
             RealtimeEvent::ConversationItemAdded(_item) => {}
             RealtimeEvent::ConversationItemDone { .. } => {}
             RealtimeEvent::HandoffRequested(_) => {}
@@ -341,6 +348,46 @@ impl ChatWidget {
         }
     }
 
+    pub(super) fn on_realtime_transcript_delta(
+        &mut self,
+        role: RealtimeTranscriptRole,
+        delta: String,
+    ) {
+        if delta.is_empty() {
+            return;
+        }
+
+        self.flush_unified_exec_wait_streak();
+
+        if let Some(cell) = self
+            .active_cell
+            .as_mut()
+            .and_then(|cell| cell.as_any_mut().downcast_mut::<RealtimeTranscriptCell>())
+            && cell.role == role
+        {
+            cell.append(&delta);
+            self.bump_active_cell_revision();
+            self.request_redraw();
+            return;
+        }
+
+        self.flush_active_cell();
+        self.active_cell = Some(Box::new(RealtimeTranscriptCell::new(role, delta)));
+        self.bump_active_cell_revision();
+        self.request_redraw();
+    }
+
+    fn flush_active_realtime_transcript(&mut self) {
+        if self
+            .active_cell
+            .as_ref()
+            .is_some_and(|cell| cell.as_any().is::<RealtimeTranscriptCell>())
+        {
+            self.flush_active_cell();
+            self.request_redraw();
+        }
+    }
+
     pub(super) fn on_realtime_conversation_closed(&mut self, ev: RealtimeConversationClosedEvent) {
         if self.realtime_conversation_uses_webrtc()
             && self.realtime_conversation.is_live()
@@ -349,6 +396,7 @@ impl ChatWidget {
             return;
         }
 
+        self.flush_active_realtime_transcript();
         let requested = self.realtime_conversation.requested_close;
         let reason = ev.reason;
         self.reset_realtime_conversation_state();

codex-rs/tui/src/chatwidget/tests.rs

@@ -172,6 +172,7 @@ pub(super) use codex_protocol::protocol::ReadOnlyAccess;
 pub(super) use codex_protocol::protocol::RealtimeConversationClosedEvent;
 pub(super) use codex_protocol::protocol::RealtimeConversationRealtimeEvent;
 pub(super) use codex_protocol::protocol::RealtimeEvent;
+pub(super) use codex_protocol::protocol::RealtimeTranscriptDelta;
 pub(super) use codex_protocol::protocol::ReviewRequest;
 pub(super) use codex_protocol::protocol::ReviewTarget;
 pub(super) use codex_protocol::protocol::SessionConfiguredEvent;

codex-rs/tui/src/chatwidget/tests/popups_and_settings.rs

@@ -22,6 +22,31 @@ async fn realtime_error_closes_without_followup_closed_info() {
     insta::assert_snapshot!(rendered.join("\n\n"), @"■ Realtime voice error: boom");
 }
 
+#[tokio::test]
+async fn realtime_output_text_delta_renders_transcript_cell() {
+    let (mut chat, _rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await;
+    chat.realtime_conversation.phase = RealtimeConversationPhase::Active;
+
+    chat.on_realtime_conversation_realtime(RealtimeConversationRealtimeEvent {
+        payload: RealtimeEvent::OutputTranscriptDelta(RealtimeTranscriptDelta {
+            delta: "hello ".to_string(),
+        }),
+    });
+    chat.on_realtime_conversation_realtime(RealtimeConversationRealtimeEvent {
+        payload: RealtimeEvent::OutputTranscriptDelta(RealtimeTranscriptDelta {
+            delta: "there".to_string(),
+        }),
+    });
+
+    let lines = chat
+        .active_cell_transcript_lines(/*width*/ 80)
+        .expect("realtime transcript cell");
+    insta::assert_snapshot!(
+        lines_to_single_string(&lines).trim_end(),
+        @"assistant: hello there"
+    );
+}
+
 #[cfg(not(target_os = "linux"))]
 #[tokio::test]
 async fn deleted_realtime_meter_uses_shared_stop_path() {

codex-rs/tui/src/history_cell.rs

@@ -489,6 +489,98 @@ impl HistoryCell for AgentMessageCell {
     }
 }
 
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub(crate) enum RealtimeTranscriptRole {
+    User,
+    Assistant,
+}
+
+impl RealtimeTranscriptRole {
+    pub(crate) fn from_name(role: &str) -> Option<Self> {
+        match role {
+            "user" => Some(Self::User),
+            "assistant" => Some(Self::Assistant),
+            _ => None,
+        }
+    }
+}
+
+#[derive(Debug)]
+pub(crate) struct RealtimeTranscriptCell {
+    pub(crate) role: RealtimeTranscriptRole,
+    text: String,
+}
+
+impl RealtimeTranscriptCell {
+    pub(crate) fn new(role: RealtimeTranscriptRole, text: String) -> Self {
+        Self { role, text }
+    }
+
+    pub(crate) fn append(&mut self, delta: &str) {
+        self.text.push_str(delta);
+    }
+}
+
+impl HistoryCell for RealtimeTranscriptCell {
+    fn display_lines(&self, width: u16) -> Vec<Line<'static>> {
+        let role_name = match self.role {
+            RealtimeTranscriptRole::User => "user",
+            RealtimeTranscriptRole::Assistant => "assistant",
+        };
+        let style = Style::default().dim().italic();
+        let text = match self.role {
+            RealtimeTranscriptRole::User => self.text.clone(),
+            RealtimeTranscriptRole::Assistant => clean_assistant_realtime_transcript(&self.text),
+        };
+        let line = Line::from(text).style(style);
+        adaptive_wrap_lines(
+            &[line],
+            RtOptions::new(width as usize)
+                .initial_indent(vec![format!("{role_name}: ").dim().italic()].into())
+                .subsequent_indent("  ".dim().italic().into()),
+        )
+    }
+}
+
+fn clean_assistant_realtime_transcript(text: &str) -> String {
+    let text = strip_realtime_code_fence(text);
+    serde_json::from_str::<serde_json::Value>(text)
+        .ok()
+        .and_then(|value| {
+            value
+                .as_object()
+                .and_then(|object| object.get("response"))
+                .and_then(serde_json::Value::as_str)
+                .map(str::to_string)
+        })
+        .unwrap_or_else(|| text.to_string())
+}
+
+fn strip_realtime_code_fence(text: &str) -> &str {
+    let text = text.trim();
+    let Some(mut rest) = text.strip_prefix("```") else {
+        return text;
+    };
+
+    let has_language = rest.chars().next().is_some_and(|ch| !ch.is_whitespace());
+    rest = rest.trim_start();
+    if has_language && let Some((first_line, remaining)) = rest.split_once('\n') {
+        let first_line = first_line.trim();
+        if !first_line.is_empty()
+            && first_line
+                .chars()
+                .all(|ch| ch.is_ascii_alphanumeric() || ch == '-' || ch == '_')
+        {
+            rest = remaining;
+        }
+    }
+
+    rest.trim()
+        .strip_suffix("```")
+        .map(str::trim)
+        .unwrap_or_else(|| rest.trim())
+}
+
 #[derive(Debug)]
 pub(crate) struct PlainHistoryCell {
     lines: Vec<Line<'static>>,
@@ -2934,6 +3026,32 @@ mod tests {
         render_lines(&cell.transcript_lines(u16::MAX))
     }
 
+    #[test]
+    fn realtime_transcript_cell_strips_assistant_code_fence() {
+        let cell = RealtimeTranscriptCell::new(
+            RealtimeTranscriptRole::Assistant,
+            "``` \nHey!".to_string(),
+        );
+
+        assert_eq!(
+            render_lines(&cell.display_lines(/*width*/ 80)),
+            vec!["assistant: Hey!"]
+        );
+    }
+
+    #[test]
+    fn realtime_transcript_cell_unwraps_assistant_response_json() {
+        let cell = RealtimeTranscriptCell::new(
+            RealtimeTranscriptRole::Assistant,
+            r#"{"response":"I'm doing well."}"#.to_string(),
+        );
+
+        assert_eq!(
+            render_lines(&cell.display_lines(/*width*/ 80)),
+            vec!["assistant: I'm doing well."]
+        );
+    }
+
     fn image_block(data: &str) -> serde_json::Value {
         serde_json::to_value(Content::image(data.to_string(), "image/png"))
             .expect("image content should serialize")