mirror of
https://github.com/openai/codex.git
synced 2026-05-04 03:16:31 +00:00
Support original-detail metadata on MCP image outputs (#17714)
## Summary - honor `_meta["codex/imageDetail"] == "original"` on MCP image content and map it to `detail: "original"` where supported - strip that detail back out when the active model does not support original-detail image inputs - update code-mode `image(...)` to accept individual MCP image blocks - teach `js_repl` / `codex.emitImage(...)` to preserve the same hint from raw MCP image outputs - document the new `_meta` contract and add generic RMCP-backed coverage across protocol, core, code-mode, and js_repl paths
This commit is contained in:
committed by
GitHub
parent
17d94bd1e3
commit
9e2fc31854
@@ -1362,6 +1362,8 @@ impl CallToolResult {
|
||||
fn convert_mcp_content_to_items(
|
||||
contents: &[serde_json::Value],
|
||||
) -> Option<Vec<FunctionCallOutputContentItem>> {
|
||||
const CODEX_IMAGE_DETAIL_META_KEY: &str = "codex/imageDetail";
|
||||
|
||||
#[derive(serde::Deserialize)]
|
||||
#[serde(tag = "type")]
|
||||
enum McpContent {
|
||||
@@ -1372,6 +1374,8 @@ fn convert_mcp_content_to_items(
|
||||
data: String,
|
||||
#[serde(rename = "mimeType", alias = "mime_type")]
|
||||
mime_type: Option<String>,
|
||||
#[serde(rename = "_meta", default)]
|
||||
meta: Option<serde_json::Value>,
|
||||
},
|
||||
#[serde(other)]
|
||||
Unknown,
|
||||
@@ -1383,7 +1387,11 @@ fn convert_mcp_content_to_items(
|
||||
for content in contents {
|
||||
let item = match serde_json::from_value::<McpContent>(content.clone()) {
|
||||
Ok(McpContent::Text { text }) => FunctionCallOutputContentItem::InputText { text },
|
||||
Ok(McpContent::Image { data, mime_type }) => {
|
||||
Ok(McpContent::Image {
|
||||
data,
|
||||
mime_type,
|
||||
meta,
|
||||
}) => {
|
||||
saw_image = true;
|
||||
let image_url = if data.starts_with("data:") {
|
||||
data
|
||||
@@ -1393,7 +1401,15 @@ fn convert_mcp_content_to_items(
|
||||
};
|
||||
FunctionCallOutputContentItem::InputImage {
|
||||
image_url,
|
||||
detail: None,
|
||||
detail: meta
|
||||
.as_ref()
|
||||
.and_then(serde_json::Value::as_object)
|
||||
.and_then(|meta| meta.get(CODEX_IMAGE_DETAIL_META_KEY))
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.and_then(|detail| match detail {
|
||||
"original" => Some(ImageDetail::Original),
|
||||
_ => None,
|
||||
}),
|
||||
}
|
||||
}
|
||||
Ok(McpContent::Unknown) | Err(_) => FunctionCallOutputContentItem::InputText {
|
||||
@@ -2264,6 +2280,70 @@ mod tests {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn preserves_original_detail_metadata_on_mcp_images() -> Result<()> {
|
||||
let call_tool_result = CallToolResult {
|
||||
content: vec![serde_json::json!({
|
||||
"type": "image",
|
||||
"data": "BASE64",
|
||||
"mimeType": "image/png",
|
||||
"_meta": {
|
||||
"codex/imageDetail": "original",
|
||||
},
|
||||
})],
|
||||
structured_content: None,
|
||||
is_error: Some(false),
|
||||
meta: None,
|
||||
};
|
||||
|
||||
let payload = call_tool_result.into_function_call_output_payload();
|
||||
let Some(items) = payload.content_items() else {
|
||||
panic!("expected content items");
|
||||
};
|
||||
let items = items.to_vec();
|
||||
assert_eq!(
|
||||
items,
|
||||
vec![FunctionCallOutputContentItem::InputImage {
|
||||
image_url: "data:image/png;base64,BASE64".into(),
|
||||
detail: Some(ImageDetail::Original),
|
||||
}]
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ignores_unknown_mcp_image_detail_metadata() -> Result<()> {
|
||||
let call_tool_result = CallToolResult {
|
||||
content: vec![serde_json::json!({
|
||||
"type": "image",
|
||||
"data": "BASE64",
|
||||
"mimeType": "image/png",
|
||||
"_meta": {
|
||||
"codex/imageDetail": "high",
|
||||
},
|
||||
})],
|
||||
structured_content: None,
|
||||
is_error: Some(false),
|
||||
meta: None,
|
||||
};
|
||||
|
||||
let payload = call_tool_result.into_function_call_output_payload();
|
||||
let Some(items) = payload.content_items() else {
|
||||
panic!("expected content items");
|
||||
};
|
||||
let items = items.to_vec();
|
||||
assert_eq!(
|
||||
items,
|
||||
vec![FunctionCallOutputContentItem::InputImage {
|
||||
image_url: "data:image/png;base64,BASE64".into(),
|
||||
detail: None,
|
||||
}]
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deserializes_array_payload_into_items() -> Result<()> {
|
||||
let json = r#"[
|
||||
|
||||
Reference in New Issue
Block a user