clone/codex
1
0
Fork 0
mirror of https://github.com/openai/codex.git synced 2026-04-30 17:36:40 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add WebRTC transport to realtime start (#16960)

Browse Source 
Adds WebRTC startup to the experimental app-server
`thread/realtime/start` method with an optional transport enum. The
websocket path remains the default; WebRTC offers create the realtime
session through the shared start flow and emit the answer SDP via
`thread/realtime/sdp`.

---------

Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
Ahmed Ibrahim
2026-04-07 15:43:38 -07:00
committed by GitHub
parent 6c36e7d688
commit fb3dcfde1d
42 changed files with 1574 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
codex-rs/app-server-protocol/schema/json/ClientRequest.json
View File

@@ -2857,6 +2857,48 @@
],
"type": "object"
},
"ThreadRealtimeStartTransport": {
"description": "EXPERIMENTAL - transport used by thread realtime.",
"oneOf": [
{
"properties": {
"type": {
"enum": [
"websocket"
],
"title": "WebsocketThreadRealtimeStartTransportType",
"type": "string"
}
},
"required": [
"type"
],
"title": "WebsocketThreadRealtimeStartTransport",
"type": "object"
},
{
"properties": {
"sdp": {
"description": "SDP offer generated by a WebRTC RTCPeerConnection after configuring audio and the realtime events data channel.",
"type": "string"
},
"type": {
"enum": [
"webrtc"
],
"title": "WebrtcThreadRealtimeStartTransportType",
"type": "string"
}
},
"required": [
"sdp",
"type"
],
"title": "WebrtcThreadRealtimeStartTransport",
"type": "object"
}
]
},
"ThreadResumeParams": {
"description": "There are three ways to resume a thread: 1. By thread_id: load the thread from disk by thread_id and resume it. 2. By history: instantiate the thread from memory and resume it. 3. By path: load the thread from disk by path and resume it.\n\nThe precedence is: history > path > thread_id. If using history or path, the thread_id param will be ignored.\n\nPrefer using thread_id whenever possible.",
"properties": {

36
codex-rs/app-server-protocol/schema/json/ServerNotification.json
View File

@@ -3304,6 +3304,22 @@
],
"type": "object"
},
"ThreadRealtimeSdpNotification": {
"description": "EXPERIMENTAL - emitted with the remote SDP for a WebRTC realtime session.",
"properties": {
"sdp": {
"type": "string"
},
"threadId": {
"type": "string"
}
},
"required": [
"sdp",
"threadId"
],
"type": "object"
},
"ThreadRealtimeStartedNotification": {
"description": "EXPERIMENTAL - emitted when thread realtime startup is accepted.",
"properties": {
@@ -4927,6 +4943,26 @@
"title": "Thread/realtime/outputAudio/deltaNotification",
"type": "object"
},
{
"properties": {
"method": {
"enum": [
"thread/realtime/sdp"
],
"title": "Thread/realtime/sdpNotificationMethod",
"type": "string"
},
"params": {
"$ref": "#/definitions/ThreadRealtimeSdpNotification"
}
},
"required": [
"method",
"params"
],
"title": "Thread/realtime/sdpNotification",
"type": "object"
},
{
"properties": {
"method": {

80
codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json
View File

@@ -4335,6 +4335,26 @@
"title": "Thread/realtime/outputAudio/deltaNotification",
"type": "object"
},
{
"properties": {
"method": {
"enum": [
"thread/realtime/sdp"
],
"title": "Thread/realtime/sdpNotificationMethod",
"type": "string"
},
"params": {
"$ref": "#/definitions/v2/ThreadRealtimeSdpNotification"
}
},
"required": [
"method",
"params"
],
"title": "Thread/realtime/sdpNotification",
"type": "object"
},
{
"properties": {
"method": {
@@ -13655,6 +13675,66 @@
"title": "ThreadRealtimeOutputAudioDeltaNotification",
"type": "object"
},
"ThreadRealtimeSdpNotification": {
"$schema": "http://json-schema.org/draft-07/schema#",
"description": "EXPERIMENTAL - emitted with the remote SDP for a WebRTC realtime session.",
"properties": {
"sdp": {
"type": "string"
},
"threadId": {
"type": "string"
}
},
"required": [
"sdp",
"threadId"
],
"title": "ThreadRealtimeSdpNotification",
"type": "object"
},
"ThreadRealtimeStartTransport": {
"description": "EXPERIMENTAL - transport used by thread realtime.",
"oneOf": [
{
"properties": {
"type": {
"enum": [
"websocket"
],
"title": "WebsocketThreadRealtimeStartTransportType",
"type": "string"
}
},
"required": [
"type"
],
"title": "WebsocketThreadRealtimeStartTransport",
"type": "object"
},
{
"properties": {
"sdp": {
"description": "SDP offer generated by a WebRTC RTCPeerConnection after configuring audio and the realtime events data channel.",
"type": "string"
},
"type": {
"enum": [
"webrtc"
],
"title": "WebrtcThreadRealtimeStartTransportType",
"type": "string"
}
},
"required": [
"sdp",
"type"
],
"title": "WebrtcThreadRealtimeStartTransport",
"type": "object"
}
]
},
"ThreadRealtimeStartedNotification": {
"$schema": "http://json-schema.org/draft-07/schema#",
"description": "EXPERIMENTAL - emitted when thread realtime startup is accepted.",

80
codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json
View File

@@ -9436,6 +9436,26 @@
"title": "Thread/realtime/outputAudio/deltaNotification",
"type": "object"
},
{
"properties": {
"method": {
"enum": [
"thread/realtime/sdp"
],
"title": "Thread/realtime/sdpNotificationMethod",
"type": "string"
},
"params": {
"$ref": "#/definitions/ThreadRealtimeSdpNotification"
}
},
"required": [
"method",
"params"
],
"title": "Thread/realtime/sdpNotification",
"type": "object"
},
{
"properties": {
"method": {
@@ -11510,6 +11530,66 @@
"title": "ThreadRealtimeOutputAudioDeltaNotification",
"type": "object"
},
"ThreadRealtimeSdpNotification": {
"$schema": "http://json-schema.org/draft-07/schema#",
"description": "EXPERIMENTAL - emitted with the remote SDP for a WebRTC realtime session.",
"properties": {
"sdp": {
"type": "string"
},
"threadId": {
"type": "string"
}
},
"required": [
"sdp",
"threadId"
],
"title": "ThreadRealtimeSdpNotification",
"type": "object"
},
"ThreadRealtimeStartTransport": {
"description": "EXPERIMENTAL - transport used by thread realtime.",
"oneOf": [
{
"properties": {
"type": {
"enum": [
"websocket"
],
"title": "WebsocketThreadRealtimeStartTransportType",
"type": "string"
}
},
"required": [
"type"
],
"title": "WebsocketThreadRealtimeStartTransport",
"type": "object"
},
{
"properties": {
"sdp": {
"description": "SDP offer generated by a WebRTC RTCPeerConnection after configuring audio and the realtime events data channel.",
"type": "string"
},
"type": {
"enum": [
"webrtc"
],
"title": "WebrtcThreadRealtimeStartTransportType",
"type": "string"
}
},
"required": [
"sdp",
"type"
],
"title": "WebrtcThreadRealtimeStartTransport",
"type": "object"
}
]
},
"ThreadRealtimeStartedNotification": {
"$schema": "http://json-schema.org/draft-07/schema#",
"description": "EXPERIMENTAL - emitted when thread realtime startup is accepted.",

18
codex-rs/app-server-protocol/schema/json/v2/ThreadRealtimeSdpNotification.json Normal file
View File

@@ -0,0 +1,18 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"description": "EXPERIMENTAL - emitted with the remote SDP for a WebRTC realtime session.",
"properties": {
"sdp": {
"type": "string"
},
"threadId": {
"type": "string"
}
},
"required": [
"sdp",
"threadId"
],
"title": "ThreadRealtimeSdpNotification",
"type": "object"
}

3
codex-rs/app-server-protocol/schema/typescript/ServerNotification.ts
View File

@@ -41,6 +41,7 @@ import type { ThreadRealtimeClosedNotification } from "./v2/ThreadRealtimeClosed
import type { ThreadRealtimeErrorNotification } from "./v2/ThreadRealtimeErrorNotification";
import type { ThreadRealtimeItemAddedNotification } from "./v2/ThreadRealtimeItemAddedNotification";
import type { ThreadRealtimeOutputAudioDeltaNotification } from "./v2/ThreadRealtimeOutputAudioDeltaNotification";
import type { ThreadRealtimeSdpNotification } from "./v2/ThreadRealtimeSdpNotification";
import type { ThreadRealtimeStartedNotification } from "./v2/ThreadRealtimeStartedNotification";
import type { ThreadRealtimeTranscriptUpdatedNotification } from "./v2/ThreadRealtimeTranscriptUpdatedNotification";
import type { ThreadStartedNotification } from "./v2/ThreadStartedNotification";
@@ -57,4 +58,4 @@ import type { WindowsWorldWritableWarningNotification } from "./v2/WindowsWorldW
/**
* Notification sent from the server to the client.
*/
export type ServerNotification = { "method": "error", "params": ErrorNotification } | { "method": "thread/started", "params": ThreadStartedNotification } | { "method": "thread/status/changed", "params": ThreadStatusChangedNotification } | { "method": "thread/archived", "params": ThreadArchivedNotification } | { "method": "thread/unarchived", "params": ThreadUnarchivedNotification } | { "method": "thread/closed", "params": ThreadClosedNotification } | { "method": "skills/changed", "params": SkillsChangedNotification } | { "method": "thread/name/updated", "params": ThreadNameUpdatedNotification } | { "method": "thread/tokenUsage/updated", "params": ThreadTokenUsageUpdatedNotification } | { "method": "turn/started", "params": TurnStartedNotification } | { "method": "hook/started", "params": HookStartedNotification } | { "method": "turn/completed", "params": TurnCompletedNotification } | { "method": "hook/completed", "params": HookCompletedNotification } | { "method": "turn/diff/updated", "params": TurnDiffUpdatedNotification } | { "method": "turn/plan/updated", "params": TurnPlanUpdatedNotification } | { "method": "item/started", "params": ItemStartedNotification } | { "method": "item/autoApprovalReview/started", "params": ItemGuardianApprovalReviewStartedNotification } | { "method": "item/autoApprovalReview/completed", "params": ItemGuardianApprovalReviewCompletedNotification } | { "method": "item/completed", "params": ItemCompletedNotification } | { "method": "rawResponseItem/completed", "params": RawResponseItemCompletedNotification } | { "method": "item/agentMessage/delta", "params": AgentMessageDeltaNotification } | { "method": "item/plan/delta", "params": PlanDeltaNotification } | { "method": "command/exec/outputDelta", "params": CommandExecOutputDeltaNotification } | { "method": "item/commandExecution/outputDelta", "params": CommandExecutionOutputDeltaNotification } | { "method": "item/commandExecution/terminalInteraction", "params": TerminalInteractionNotification } | { "method": "item/fileChange/outputDelta", "params": FileChangeOutputDeltaNotification } | { "method": "serverRequest/resolved", "params": ServerRequestResolvedNotification } | { "method": "item/mcpToolCall/progress", "params": McpToolCallProgressNotification } | { "method": "mcpServer/oauthLogin/completed", "params": McpServerOauthLoginCompletedNotification } | { "method": "mcpServer/startupStatus/updated", "params": McpServerStatusUpdatedNotification } | { "method": "account/updated", "params": AccountUpdatedNotification } | { "method": "account/rateLimits/updated", "params": AccountRateLimitsUpdatedNotification } | { "method": "app/list/updated", "params": AppListUpdatedNotification } | { "method": "fs/changed", "params": FsChangedNotification } | { "method": "item/reasoning/summaryTextDelta", "params": ReasoningSummaryTextDeltaNotification } | { "method": "item/reasoning/summaryPartAdded", "params": ReasoningSummaryPartAddedNotification } | { "method": "item/reasoning/textDelta", "params": ReasoningTextDeltaNotification } | { "method": "thread/compacted", "params": ContextCompactedNotification } | { "method": "model/rerouted", "params": ModelReroutedNotification } | { "method": "deprecationNotice", "params": DeprecationNoticeNotification } | { "method": "configWarning", "params": ConfigWarningNotification } | { "method": "fuzzyFileSearch/sessionUpdated", "params": FuzzyFileSearchSessionUpdatedNotification } | { "method": "fuzzyFileSearch/sessionCompleted", "params": FuzzyFileSearchSessionCompletedNotification } | { "method": "thread/realtime/started", "params": ThreadRealtimeStartedNotification } | { "method": "thread/realtime/itemAdded", "params": ThreadRealtimeItemAddedNotification } | { "method": "thread/realtime/transcriptUpdated", "params": ThreadRealtimeTranscriptUpdatedNotification } | { "method": "thread/realtime/outputAudio/delta", "params": ThreadRealtimeOutputAudioDeltaNotification } | { "method": "thread/realtime/error", "params": ThreadRealtimeErrorNotification } | { "method": "thread/realtime/closed", "params": ThreadRealtimeClosedNotification } | { "method": "windows/worldWritableWarning", "params": WindowsWorldWritableWarningNotification } | { "method": "windowsSandbox/setupCompleted", "params": WindowsSandboxSetupCompletedNotification } | { "method": "account/login/completed", "params": AccountLoginCompletedNotification };
export type ServerNotification = { "method": "error", "params": ErrorNotification } | { "method": "thread/started", "params": ThreadStartedNotification } | { "method": "thread/status/changed", "params": ThreadStatusChangedNotification } | { "method": "thread/archived", "params": ThreadArchivedNotification } | { "method": "thread/unarchived", "params": ThreadUnarchivedNotification } | { "method": "thread/closed", "params": ThreadClosedNotification } | { "method": "skills/changed", "params": SkillsChangedNotification } | { "method": "thread/name/updated", "params": ThreadNameUpdatedNotification } | { "method": "thread/tokenUsage/updated", "params": ThreadTokenUsageUpdatedNotification } | { "method": "turn/started", "params": TurnStartedNotification } | { "method": "hook/started", "params": HookStartedNotification } | { "method": "turn/completed", "params": TurnCompletedNotification } | { "method": "hook/completed", "params": HookCompletedNotification } | { "method": "turn/diff/updated", "params": TurnDiffUpdatedNotification } | { "method": "turn/plan/updated", "params": TurnPlanUpdatedNotification } | { "method": "item/started", "params": ItemStartedNotification } | { "method": "item/autoApprovalReview/started", "params": ItemGuardianApprovalReviewStartedNotification } | { "method": "item/autoApprovalReview/completed", "params": ItemGuardianApprovalReviewCompletedNotification } | { "method": "item/completed", "params": ItemCompletedNotification } | { "method": "rawResponseItem/completed", "params": RawResponseItemCompletedNotification } | { "method": "item/agentMessage/delta", "params": AgentMessageDeltaNotification } | { "method": "item/plan/delta", "params": PlanDeltaNotification } | { "method": "command/exec/outputDelta", "params": CommandExecOutputDeltaNotification } | { "method": "item/commandExecution/outputDelta", "params": CommandExecutionOutputDeltaNotification } | { "method": "item/commandExecution/terminalInteraction", "params": TerminalInteractionNotification } | { "method": "item/fileChange/outputDelta", "params": FileChangeOutputDeltaNotification } | { "method": "serverRequest/resolved", "params": ServerRequestResolvedNotification } | { "method": "item/mcpToolCall/progress", "params": McpToolCallProgressNotification } | { "method": "mcpServer/oauthLogin/completed", "params": McpServerOauthLoginCompletedNotification } | { "method": "mcpServer/startupStatus/updated", "params": McpServerStatusUpdatedNotification } | { "method": "account/updated", "params": AccountUpdatedNotification } | { "method": "account/rateLimits/updated", "params": AccountRateLimitsUpdatedNotification } | { "method": "app/list/updated", "params": AppListUpdatedNotification } | { "method": "fs/changed", "params": FsChangedNotification } | { "method": "item/reasoning/summaryTextDelta", "params": ReasoningSummaryTextDeltaNotification } | { "method": "item/reasoning/summaryPartAdded", "params": ReasoningSummaryPartAddedNotification } | { "method": "item/reasoning/textDelta", "params": ReasoningTextDeltaNotification } | { "method": "thread/compacted", "params": ContextCompactedNotification } | { "method": "model/rerouted", "params": ModelReroutedNotification } | { "method": "deprecationNotice", "params": DeprecationNoticeNotification } | { "method": "configWarning", "params": ConfigWarningNotification } | { "method": "fuzzyFileSearch/sessionUpdated", "params": FuzzyFileSearchSessionUpdatedNotification } | { "method": "fuzzyFileSearch/sessionCompleted", "params": FuzzyFileSearchSessionCompletedNotification } | { "method": "thread/realtime/started", "params": ThreadRealtimeStartedNotification } | { "method": "thread/realtime/itemAdded", "params": ThreadRealtimeItemAddedNotification } | { "method": "thread/realtime/transcriptUpdated", "params": ThreadRealtimeTranscriptUpdatedNotification } | { "method": "thread/realtime/outputAudio/delta", "params": ThreadRealtimeOutputAudioDeltaNotification } | { "method": "thread/realtime/sdp", "params": ThreadRealtimeSdpNotification } | { "method": "thread/realtime/error", "params": ThreadRealtimeErrorNotification } | { "method": "thread/realtime/closed", "params": ThreadRealtimeClosedNotification } | { "method": "windows/worldWritableWarning", "params": WindowsWorldWritableWarningNotification } | { "method": "windowsSandbox/setupCompleted", "params": WindowsSandboxSetupCompletedNotification } | { "method": "account/login/completed", "params": AccountLoginCompletedNotification };

8
codex-rs/app-server-protocol/schema/typescript/v2/ThreadRealtimeSdpNotification.ts Normal file
View File

@@ -0,0 +1,8 @@
// GENERATED CODE! DO NOT MODIFY BY HAND!
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
/**
* EXPERIMENTAL - emitted with the remote SDP for a WebRTC realtime session.
*/
export type ThreadRealtimeSdpNotification = { threadId: string, sdp: string, };

13
codex-rs/app-server-protocol/schema/typescript/v2/ThreadRealtimeStartTransport.ts Normal file
View File

@@ -0,0 +1,13 @@
// GENERATED CODE! DO NOT MODIFY BY HAND!
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
/**
* EXPERIMENTAL - transport used by thread realtime.
*/
export type ThreadRealtimeStartTransport = { "type": "websocket" } | { "type": "webrtc",
/**
* SDP offer generated by a WebRTC RTCPeerConnection after configuring audio and the
* realtime events data channel.
*/
sdp: string, };

2
codex-rs/app-server-protocol/schema/typescript/v2/index.ts
View File

@@ -294,6 +294,8 @@ export type { ThreadRealtimeClosedNotification } from "./ThreadRealtimeClosedNot
export type { ThreadRealtimeErrorNotification } from "./ThreadRealtimeErrorNotification";
export type { ThreadRealtimeItemAddedNotification } from "./ThreadRealtimeItemAddedNotification";
export type { ThreadRealtimeOutputAudioDeltaNotification } from "./ThreadRealtimeOutputAudioDeltaNotification";
export type { ThreadRealtimeSdpNotification } from "./ThreadRealtimeSdpNotification";
export type { ThreadRealtimeStartTransport } from "./ThreadRealtimeStartTransport";
export type { ThreadRealtimeStartedNotification } from "./ThreadRealtimeStartedNotification";
export type { ThreadRealtimeTranscriptUpdatedNotification } from "./ThreadRealtimeTranscriptUpdatedNotification";
export type { ThreadResumeParams } from "./ThreadResumeParams";