Support original-detail metadata on MCP image outputs (#17714)

## Summary
- honor `_meta["codex/imageDetail"] == "original"` on MCP image content
and map it to `detail: "original"` where supported
- strip that detail back out when the active model does not support
original-detail image inputs
- update code-mode `image(...)` to accept individual MCP image blocks
- teach `js_repl` / `codex.emitImage(...)` to preserve the same hint
from raw MCP image outputs
- document the new `_meta` contract and add generic RMCP-backed coverage
across protocol, core, code-mode, and js_repl paths
This commit is contained in:
Curtis 'Fjord' Hawthorne
2026-04-15 14:43:33 -07:00
committed by GitHub
parent 17d94bd1e3
commit 9e2fc31854
20 changed files with 905 additions and 368 deletions

View File

@@ -178,7 +178,16 @@ async fn run_code_mode_turn_with_rmcp(
prompt: &str,
code: &str,
) -> Result<(TestCodex, ResponseMock)> {
run_code_mode_turn_with_rmcp_mode(server, prompt, code, /*code_mode_only*/ false).await
run_code_mode_turn_with_rmcp_model(server, prompt, code, "test-gpt-5.1-codex").await
}
async fn run_code_mode_turn_with_rmcp_model(
server: &MockServer,
prompt: &str,
code: &str,
model: &'static str,
) -> Result<(TestCodex, ResponseMock)> {
run_code_mode_turn_with_rmcp_config(server, prompt, code, model, /*code_mode_only*/ false).await
}
async fn run_code_mode_turn_with_rmcp_mode(
@@ -187,48 +196,57 @@ async fn run_code_mode_turn_with_rmcp_mode(
code: &str,
code_mode_only: bool,
) -> Result<(TestCodex, ResponseMock)> {
let rmcp_test_server_bin = stdio_server_bin()?;
let mut builder = test_codex()
.with_model("test-gpt-5.1-codex")
.with_config(move |config| {
let _ = if code_mode_only {
config.features.enable(Feature::CodeModeOnly)
} else {
config.features.enable(Feature::CodeMode)
};
run_code_mode_turn_with_rmcp_config(server, prompt, code, "test-gpt-5.1-codex", code_mode_only)
.await
}
let mut servers = config.mcp_servers.get().clone();
servers.insert(
"rmcp".to_string(),
McpServerConfig {
transport: McpServerTransportConfig::Stdio {
command: rmcp_test_server_bin,
args: Vec::new(),
env: Some(HashMap::from([(
"MCP_TEST_VALUE".to_string(),
"propagated-env".to_string(),
)])),
env_vars: Vec::new(),
cwd: None,
},
enabled: true,
required: false,
supports_parallel_tool_calls: false,
disabled_reason: None,
startup_timeout_sec: Some(Duration::from_secs(10)),
tool_timeout_sec: None,
enabled_tools: None,
disabled_tools: None,
scopes: None,
oauth_resource: None,
tools: HashMap::new(),
async fn run_code_mode_turn_with_rmcp_config(
server: &MockServer,
prompt: &str,
code: &str,
model: &'static str,
code_mode_only: bool,
) -> Result<(TestCodex, ResponseMock)> {
let rmcp_test_server_bin = stdio_server_bin()?;
let mut builder = test_codex().with_model(model).with_config(move |config| {
let _ = if code_mode_only {
config.features.enable(Feature::CodeModeOnly)
} else {
config.features.enable(Feature::CodeMode)
};
let mut servers = config.mcp_servers.get().clone();
servers.insert(
"rmcp".to_string(),
McpServerConfig {
transport: McpServerTransportConfig::Stdio {
command: rmcp_test_server_bin,
args: Vec::new(),
env: Some(HashMap::from([(
"MCP_TEST_VALUE".to_string(),
"propagated-env".to_string(),
)])),
env_vars: Vec::new(),
cwd: None,
},
);
config
.mcp_servers
.set(servers)
.expect("test mcp servers should accept any configuration");
});
enabled: true,
required: false,
supports_parallel_tool_calls: false,
disabled_reason: None,
startup_timeout_sec: Some(Duration::from_secs(10)),
tool_timeout_sec: None,
enabled_tools: None,
disabled_tools: None,
scopes: None,
oauth_resource: None,
tools: HashMap::new(),
},
);
config
.mcp_servers
.set(servers)
.expect("test mcp servers should accept any configuration");
});
let test = builder.build(server).await?;
responses::mount_sse_once(
@@ -1919,6 +1937,62 @@ image(out);
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn code_mode_can_use_mcp_image_result_with_image_helper() -> Result<()> {
skip_if_no_network!(Ok(()));
let server = responses::start_mock_server().await;
let code = r#"
const out = await tools.mcp__rmcp__image_scenario({
scenario: "image_only_original_detail",
});
const imageItem = out.content.find((item) => item.type === "image");
image(imageItem);
"#;
let (_test, second_mock) = run_code_mode_turn_with_rmcp_model(
&server,
"use exec to call the rmcp image scenario tool and emit its image output",
code,
"gpt-5.3-codex",
)
.await?;
let req = second_mock.single_request();
let items = custom_tool_output_items(&req, "call-1");
let (_, success) = custom_tool_output_body_and_success(&req, "call-1");
assert_ne!(
success,
Some(false),
"code_mode mcp image scenario call failed unexpectedly"
);
assert_eq!(items.len(), 2);
assert_regex_match(
concat!(
r"(?s)\A",
r"Script completed\nWall time \d+\.\d seconds\nOutput:\n\z"
),
text_item(&items, /*index*/ 0),
);
assert_eq!(
items[1].get("type").and_then(Value::as_str),
Some("input_image")
);
let emitted_image_url = items[1]
.get("image_url")
.and_then(Value::as_str)
.expect("image helper should emit an input_image item with image_url");
assert!(emitted_image_url.starts_with("data:image/png;base64,"));
assert_eq!(
items[1].get("detail").and_then(Value::as_str),
Some("original")
);
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn code_mode_can_apply_patch_via_nested_tool() -> Result<()> {
skip_if_no_network!(Ok(()));