clone/codex
1
0
Fork 0
mirror of https://github.com/openai/codex.git synced 2026-04-27 16:15:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add text element metadata to protocol, app server, and core (#9331)

Browse Source 
The second part of breaking up PR
https://github.com/openai/codex/pull/9116

Summary:

- Add `TextElement` / `ByteRange` to protocol user inputs and user
message events with defaults.
- Thread `text_elements` through app-server v1/v2 request handling and
history rebuild.
- Preserve UI metadata only in user input/events (not `ContentItem`)
while keeping local image attachments in user events for rehydration.

Details:

- Protocol: `UserInput::Text` carries `text_elements`;
`UserMessageEvent` carries `text_elements` + `local_images`.
Serialization includes empty vectors for backward compatibility.
- app-server-protocol: v1 defines `V1TextElement` / `V1ByteRange` in
camelCase with conversions; v2 uses its own camelCase wrapper.
- app-server: v1/v2 input mapping includes `text_elements`; thread
history rebuilds include them.
- Core: user event emission preserves UI metadata while model history
stays clean; history replay round-trips the metadata.
This commit is contained in:
charley-oai
2026-01-15 17:26:41 -08:00
committed by GitHub
parent 004a74940a
commit 1fa8350ae7
18 changed files with 416 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
codex-rs/app-server/tests/common/lib.rs
View File

@@ -29,6 +29,7 @@ pub use responses::create_exec_command_sse_response;
pub use responses::create_final_assistant_message_sse_response;
pub use responses::create_shell_command_sse_response;
pub use rollout::create_fake_rollout;
pub use rollout::create_fake_rollout_with_text_elements;
use serde::de::DeserializeOwned;
pub fn to_response<T: DeserializeOwned>(response: JSONRPCResponse) -> anyhow::Result<T> {

72
codex-rs/app-server/tests/common/rollout.rs
View File

@@ -87,3 +87,75 @@ pub fn create_fake_rollout(
fs::write(file_path, lines.join("\n") + "\n")?;
Ok(uuid_str)
}
pub fn create_fake_rollout_with_text_elements(
codex_home: &Path,
filename_ts: &str,
meta_rfc3339: &str,
preview: &str,
text_elements: Vec<serde_json::Value>,
model_provider: Option<&str>,
git_info: Option<GitInfo>,
) -> Result<String> {
let uuid = Uuid::new_v4();
let uuid_str = uuid.to_string();
let conversation_id = ThreadId::from_string(&uuid_str)?;
// sessions/YYYY/MM/DD derived from filename_ts (YYYY-MM-DDThh-mm-ss)
let year = &filename_ts[0..4];
let month = &filename_ts[5..7];
let day = &filename_ts[8..10];
let dir = codex_home.join("sessions").join(year).join(month).join(day);
fs::create_dir_all(&dir)?;
let file_path = dir.join(format!("rollout-{filename_ts}-{uuid}.jsonl"));
// Build JSONL lines
let meta = SessionMeta {
id: conversation_id,
timestamp: meta_rfc3339.to_string(),
cwd: PathBuf::from("/"),
originator: "codex".to_string(),
cli_version: "0.0.0".to_string(),
instructions: None,
source: SessionSource::Cli,
model_provider: model_provider.map(str::to_string),
};
let payload = serde_json::to_value(SessionMetaLine {
meta,
git: git_info,
})?;
let lines = [
json!( {
"timestamp": meta_rfc3339,
"type": "session_meta",
"payload": payload
})
.to_string(),
json!( {
"timestamp": meta_rfc3339,
"type":"response_item",
"payload": {
"type":"message",
"role":"user",
"content":[{"type":"input_text","text": preview}]
}
})
.to_string(),
json!( {
"timestamp": meta_rfc3339,
"type":"event_msg",
"payload": {
"type":"user_message",
"message": preview,
"text_elements": text_elements,
"local_images": []
}
})
.to_string(),
];
fs::write(file_path, lines.join("\n") + "\n")?;
Ok(uuid_str)
}

5
codex-rs/app-server/tests/suite/codex_message_processor_flow.rs
View File

@@ -114,6 +114,7 @@ async fn test_codex_jsonrpc_conversation_flow() -> Result<()> {
conversation_id,
items: vec![codex_app_server_protocol::InputItem::Text {
text: "text".to_string(),
text_elements: Vec::new(),
}],
})
.await?;
@@ -241,6 +242,7 @@ async fn test_send_user_turn_changes_approval_policy_behavior() -> Result<()> {
conversation_id,
items: vec![codex_app_server_protocol::InputItem::Text {
text: "run python".to_string(),
text_elements: Vec::new(),
}],
})
.await?;
@@ -296,6 +298,7 @@ async fn test_send_user_turn_changes_approval_policy_behavior() -> Result<()> {
conversation_id,
items: vec![codex_app_server_protocol::InputItem::Text {
text: "run python again".to_string(),
text_elements: Vec::new(),
}],
cwd: working_directory.clone(),
approval_policy: AskForApproval::Never,
@@ -405,6 +408,7 @@ async fn test_send_user_turn_updates_sandbox_and_cwd_between_turns() -> Result<(
conversation_id,
items: vec![InputItem::Text {
text: "first turn".to_string(),
text_elements: Vec::new(),
}],
cwd: first_cwd.clone(),
approval_policy: AskForApproval::Never,
@@ -437,6 +441,7 @@ async fn test_send_user_turn_updates_sandbox_and_cwd_between_turns() -> Result<(
conversation_id,
items: vec![InputItem::Text {
text: "second turn".to_string(),
text_elements: Vec::new(),
}],
cwd: second_cwd.clone(),
approval_policy: AskForApproval::Never,

1
codex-rs/app-server/tests/suite/create_thread.rs
View File

@@ -77,6 +77,7 @@ async fn test_conversation_create_and_send_message_ok() -> Result<()> {
conversation_id,
items: vec![InputItem::Text {
text: "Hello".to_string(),
text_elements: Vec::new(),
}],
})
.await?;

1
codex-rs/app-server/tests/suite/interrupt.rs
View File

@@ -105,6 +105,7 @@ async fn shell_command_interruption() -> anyhow::Result<()> {
conversation_id,
items: vec![codex_app_server_protocol::InputItem::Text {
text: "run first sleep command".to_string(),
text_elements: Vec::new(),
}],
})
.await?;

3
codex-rs/app-server/tests/suite/output_schema.rs
View File

@@ -80,6 +80,7 @@ async fn send_user_turn_accepts_output_schema_v1() -> Result<()> {
conversation_id,
items: vec![InputItem::Text {
text: "Hello".to_string(),
text_elements: Vec::new(),
}],
cwd: codex_home.path().to_path_buf(),
approval_policy: AskForApproval::Never,
@@ -181,6 +182,7 @@ async fn send_user_turn_output_schema_is_per_turn_v1() -> Result<()> {
conversation_id,
items: vec![InputItem::Text {
text: "Hello".to_string(),
text_elements: Vec::new(),
}],
cwd: codex_home.path().to_path_buf(),
approval_policy: AskForApproval::Never,
@@ -228,6 +230,7 @@ async fn send_user_turn_output_schema_is_per_turn_v1() -> Result<()> {
conversation_id,
items: vec![InputItem::Text {
text: "Hello again".to_string(),
text_elements: Vec::new(),
}],
cwd: codex_home.path().to_path_buf(),
approval_policy: AskForApproval::Never,

5
codex-rs/app-server/tests/suite/send_message.rs
View File

@@ -101,6 +101,7 @@ async fn send_message(
conversation_id,
items: vec![InputItem::Text {
text: message.to_string(),
text_elements: Vec::new(),
}],
})
.await?;
@@ -194,6 +195,7 @@ async fn test_send_message_raw_notifications_opt_in() -> Result<()> {
conversation_id,
items: vec![InputItem::Text {
text: "Hello".to_string(),
text_elements: Vec::new(),
}],
})
.await?;
@@ -245,6 +247,7 @@ async fn test_send_message_session_not_found() -> Result<()> {
conversation_id: unknown,
items: vec![InputItem::Text {
text: "ping".to_string(),
text_elements: Vec::new(),
}],
})
.await?;
@@ -425,7 +428,7 @@ fn content_texts(content: &[ContentItem]) -> Vec<&str> {
content
.iter()
.filter_map(|item| match item {
ContentItem::InputText { text } | ContentItem::OutputText { text } => {
ContentItem::InputText { text, .. } | ContentItem::OutputText { text } => {
Some(text.as_str())
}
_ => None,

17
codex-rs/app-server/tests/suite/v2/thread_resume.rs
View File

@@ -1,6 +1,6 @@
use anyhow::Result;
use app_test_support::McpProcess;
use app_test_support::create_fake_rollout;
use app_test_support::create_fake_rollout_with_text_elements;
use app_test_support::create_mock_responses_server_repeating_assistant;
use app_test_support::to_response;
use codex_app_server_protocol::JSONRPCResponse;
@@ -15,6 +15,9 @@ use codex_app_server_protocol::TurnStatus;
use codex_app_server_protocol::UserInput;
use codex_protocol::models::ContentItem;
use codex_protocol::models::ResponseItem;
use codex_protocol::user_input::ByteRange;
use codex_protocol::user_input::TextElement;
use pretty_assertions::assert_eq;
use std::path::PathBuf;
use tempfile::TempDir;
use tokio::time::timeout;
@@ -71,11 +74,19 @@ async fn thread_resume_returns_rollout_history() -> Result<()> {
create_config_toml(codex_home.path(), &server.uri())?;
let preview = "Saved user message";
let conversation_id = create_fake_rollout(
let text_elements = vec![TextElement {
byte_range: ByteRange { start: 0, end: 5 },
placeholder: Some("<note>".into()),
}];
let conversation_id = create_fake_rollout_with_text_elements(
codex_home.path(),
"2025-01-05T12-00-00",
"2025-01-05T12:00:00Z",
preview,
text_elements
.iter()
.map(|elem| serde_json::to_value(elem).expect("serialize text element"))
.collect(),
Some("mock_provider"),
None,
)?;
@@ -119,7 +130,7 @@ async fn thread_resume_returns_rollout_history() -> Result<()> {
content,
&vec![UserInput::Text {
text: preview.to_string(),
text_elements: Vec::new(),
text_elements: text_elements.clone().into_iter().map(Into::into).collect(),
}]
);
}

83
codex-rs/app-server/tests/suite/v2/turn_start.rs
View File

@@ -8,6 +8,7 @@ use app_test_support::create_mock_responses_server_sequence_unchecked;
use app_test_support::create_shell_command_sse_response;
use app_test_support::format_with_current_shell_display;
use app_test_support::to_response;
use codex_app_server_protocol::ByteRange;
use codex_app_server_protocol::ClientInfo;
use codex_app_server_protocol::CommandExecutionApprovalDecision;
use codex_app_server_protocol::CommandExecutionRequestApprovalResponse;
@@ -23,6 +24,7 @@ use codex_app_server_protocol::PatchApplyStatus;
use codex_app_server_protocol::PatchChangeKind;
use codex_app_server_protocol::RequestId;
use codex_app_server_protocol::ServerRequest;
use codex_app_server_protocol::TextElement;
use codex_app_server_protocol::ThreadItem;
use codex_app_server_protocol::ThreadStartParams;
use codex_app_server_protocol::ThreadStartResponse;
@@ -113,6 +115,87 @@ async fn turn_start_sends_originator_header() -> Result<()> {
Ok(())
}
#[tokio::test]
async fn turn_start_emits_user_message_item_with_text_elements() -> Result<()> {
let responses = vec![create_final_assistant_message_sse_response("Done")?];
let server = create_mock_responses_server_sequence_unchecked(responses).await;
let codex_home = TempDir::new()?;
create_config_toml(codex_home.path(), &server.uri(), "never")?;
let mut mcp = McpProcess::new(codex_home.path()).await?;
timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??;
let thread_req = mcp
.send_thread_start_request(ThreadStartParams {
model: Some("mock-model".to_string()),
..Default::default()
})
.await?;
let thread_resp: JSONRPCResponse = timeout(
DEFAULT_READ_TIMEOUT,
mcp.read_stream_until_response_message(RequestId::Integer(thread_req)),
)
.await??;
let ThreadStartResponse { thread, .. } = to_response::<ThreadStartResponse>(thread_resp)?;
let text_elements = vec![TextElement {
byte_range: ByteRange { start: 0, end: 5 },
placeholder: Some("<note>".to_string()),
}];
let turn_req = mcp
.send_turn_start_request(TurnStartParams {
thread_id: thread.id.clone(),
input: vec![V2UserInput::Text {
text: "Hello".to_string(),
text_elements: text_elements.clone(),
}],
..Default::default()
})
.await?;
timeout(
DEFAULT_READ_TIMEOUT,
mcp.read_stream_until_response_message(RequestId::Integer(turn_req)),
)
.await??;
let user_message_item = timeout(DEFAULT_READ_TIMEOUT, async {
loop {
let notification = mcp
.read_stream_until_notification_message("item/started")
.await?;
let params = notification.params.expect("item/started params");
let item_started: ItemStartedNotification =
serde_json::from_value(params).expect("deserialize item/started notification");
if let ThreadItem::UserMessage { .. } = item_started.item {
return Ok::<ThreadItem, anyhow::Error>(item_started.item);
}
}
})
.await??;
match user_message_item {
ThreadItem::UserMessage { content, .. } => {
assert_eq!(
content,
vec![V2UserInput::Text {
text: "Hello".to_string(),
text_elements,
}]
);
}
other => panic!("expected user message item, got {other:?}"),
}
timeout(
DEFAULT_READ_TIMEOUT,
mcp.read_stream_until_notification_message("turn/completed"),
)
.await??;
Ok(())
}
#[tokio::test]
async fn turn_start_emits_notifications_and_accepts_model_override() -> Result<()> {
// Provide a mock server and config so model wiring is valid.