mirror of
https://github.com/openai/codex.git
synced 2026-04-28 00:25:56 +00:00
Support multimodal custom tool outputs (#12948)
## Summary This changes `custom_tool_call_output` to use the same output payload shape as `function_call_output`, so freeform tools can return either plain text or structured content items. The main goal is to let `js_repl` return image content from nested `view_image` calls in its own `custom_tool_call_output`, instead of relying on a separate injected message. ## What changed - Changed `custom_tool_call_output.output` from `string` to `FunctionCallOutputPayload` - Updated freeform tool plumbing to preserve structured output bodies - Updated `js_repl` to aggregate nested tool content items and attach them to the outer `js_repl` result - Removed the old `js_repl` special case that injected `view_image` results as a separate pending user image message - Updated normalization/history/truncation paths to handle multimodal `custom_tool_call_output` - Regenerated app-server protocol schema artifacts ## Behavior Direct `view_image` calls still return a `function_call_output` with image content. When `view_image` is called inside `js_repl`, the outer `js_repl` `custom_tool_call_output` now carries: - an `input_text` item if the JS produced text output - one or more `input_image` items from nested tool results So the nested image result now stays inside the `js_repl` tool output instead of being injected as a separate message. ## Compatibility This is intended to be backward-compatible for resumed conversations. Older histories that stored `custom_tool_call_output.output` as a plain string still deserialize correctly, and older histories that used the previous injected-image-message flow also continue to resume. Added regression coverage for resuming a pre-change rollout containing: - string-valued `custom_tool_call_output` - legacy injected image message history #### [git stack](https://github.com/magus/git-stack-cli) - 👉 `1` https://github.com/openai/codex/pull/12948
This commit is contained in:
committed by
GitHub
parent
f90e97e414
commit
7e980d7db6
@@ -358,40 +358,26 @@ console.log(out.output?.body?.text ?? "");
|
||||
.await;
|
||||
|
||||
let req = mock.single_request();
|
||||
let (js_repl_output, js_repl_success) = req
|
||||
.custom_tool_call_output_content_and_success(call_id)
|
||||
.expect("custom tool output present");
|
||||
let js_repl_output = js_repl_output.expect("custom tool output text present");
|
||||
assert_ne!(
|
||||
js_repl_success,
|
||||
Some(false),
|
||||
"js_repl call failed unexpectedly: {js_repl_output}"
|
||||
let body = req.body_json();
|
||||
assert_eq!(
|
||||
image_messages(&body).len(),
|
||||
0,
|
||||
"js_repl view_image should not inject a pending input image message"
|
||||
);
|
||||
|
||||
let body = req.body_json();
|
||||
let image_messages = image_messages(&body);
|
||||
assert_eq!(
|
||||
image_messages.len(),
|
||||
1,
|
||||
"js_repl view_image should inject exactly one pending input image message"
|
||||
);
|
||||
let image_message = image_messages
|
||||
.into_iter()
|
||||
.next()
|
||||
.expect("pending input image message not included in request");
|
||||
let image_url = image_message
|
||||
.get("content")
|
||||
let custom_output = req.custom_tool_call_output(call_id);
|
||||
let output_items = custom_output
|
||||
.get("output")
|
||||
.and_then(Value::as_array)
|
||||
.and_then(|content| {
|
||||
content.iter().find_map(|span| {
|
||||
if span.get("type").and_then(Value::as_str) == Some("input_image") {
|
||||
span.get("image_url").and_then(Value::as_str)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.expect("custom_tool_call_output should be a content item array");
|
||||
let image_url = output_items
|
||||
.iter()
|
||||
.find_map(|item| {
|
||||
(item.get("type").and_then(Value::as_str) == Some("input_image"))
|
||||
.then(|| item.get("image_url").and_then(Value::as_str))
|
||||
.flatten()
|
||||
})
|
||||
.expect("image_url present");
|
||||
.expect("image_url present in js_repl custom tool output");
|
||||
assert!(
|
||||
image_url.starts_with("data:image/png;base64,"),
|
||||
"expected png data URL, got {image_url}"
|
||||
|
||||
Reference in New Issue
Block a user