Support original-detail metadata on MCP image outputs (#17714)

## Summary
- honor `_meta["codex/imageDetail"] == "original"` on MCP image content
and map it to `detail: "original"` where supported
- strip that detail back out when the active model does not support
original-detail image inputs
- update code-mode `image(...)` to accept individual MCP image blocks
- teach `js_repl` / `codex.emitImage(...)` to preserve the same hint
from raw MCP image outputs
- document the new `_meta` contract and add generic RMCP-backed coverage
across protocol, core, code-mode, and js_repl paths
This commit is contained in:
Curtis 'Fjord' Hawthorne
2026-04-15 14:43:33 -07:00
committed by GitHub
parent 17d94bd1e3
commit 9e2fc31854
20 changed files with 905 additions and 368 deletions

View File

@@ -1362,6 +1362,8 @@ impl CallToolResult {
fn convert_mcp_content_to_items(
contents: &[serde_json::Value],
) -> Option<Vec<FunctionCallOutputContentItem>> {
const CODEX_IMAGE_DETAIL_META_KEY: &str = "codex/imageDetail";
#[derive(serde::Deserialize)]
#[serde(tag = "type")]
enum McpContent {
@@ -1372,6 +1374,8 @@ fn convert_mcp_content_to_items(
data: String,
#[serde(rename = "mimeType", alias = "mime_type")]
mime_type: Option<String>,
#[serde(rename = "_meta", default)]
meta: Option<serde_json::Value>,
},
#[serde(other)]
Unknown,
@@ -1383,7 +1387,11 @@ fn convert_mcp_content_to_items(
for content in contents {
let item = match serde_json::from_value::<McpContent>(content.clone()) {
Ok(McpContent::Text { text }) => FunctionCallOutputContentItem::InputText { text },
Ok(McpContent::Image { data, mime_type }) => {
Ok(McpContent::Image {
data,
mime_type,
meta,
}) => {
saw_image = true;
let image_url = if data.starts_with("data:") {
data
@@ -1393,7 +1401,15 @@ fn convert_mcp_content_to_items(
};
FunctionCallOutputContentItem::InputImage {
image_url,
detail: None,
detail: meta
.as_ref()
.and_then(serde_json::Value::as_object)
.and_then(|meta| meta.get(CODEX_IMAGE_DETAIL_META_KEY))
.and_then(serde_json::Value::as_str)
.and_then(|detail| match detail {
"original" => Some(ImageDetail::Original),
_ => None,
}),
}
}
Ok(McpContent::Unknown) | Err(_) => FunctionCallOutputContentItem::InputText {
@@ -2264,6 +2280,70 @@ mod tests {
Ok(())
}
#[test]
fn preserves_original_detail_metadata_on_mcp_images() -> Result<()> {
let call_tool_result = CallToolResult {
content: vec![serde_json::json!({
"type": "image",
"data": "BASE64",
"mimeType": "image/png",
"_meta": {
"codex/imageDetail": "original",
},
})],
structured_content: None,
is_error: Some(false),
meta: None,
};
let payload = call_tool_result.into_function_call_output_payload();
let Some(items) = payload.content_items() else {
panic!("expected content items");
};
let items = items.to_vec();
assert_eq!(
items,
vec![FunctionCallOutputContentItem::InputImage {
image_url: "data:image/png;base64,BASE64".into(),
detail: Some(ImageDetail::Original),
}]
);
Ok(())
}
#[test]
fn ignores_unknown_mcp_image_detail_metadata() -> Result<()> {
let call_tool_result = CallToolResult {
content: vec![serde_json::json!({
"type": "image",
"data": "BASE64",
"mimeType": "image/png",
"_meta": {
"codex/imageDetail": "high",
},
})],
structured_content: None,
is_error: Some(false),
meta: None,
};
let payload = call_tool_result.into_function_call_output_payload();
let Some(items) = payload.content_items() else {
panic!("expected content items");
};
let items = items.to_vec();
assert_eq!(
items,
vec![FunctionCallOutputContentItem::InputImage {
image_url: "data:image/png;base64,BASE64".into(),
detail: None,
}]
);
Ok(())
}
#[test]
fn deserializes_array_payload_into_items() -> Result<()> {
let json = r#"[