From 01e102ef8988467abd82d8b2b159ff18cad52d2e Mon Sep 17 00:00:00 2001 From: Curtis 'Fjord' Hawthorne Date: Mon, 11 May 2026 11:49:17 -0700 Subject: [PATCH] Add code mode forward_output helper --- codex-rs/code-mode/src/description.rs | 1 + codex-rs/code-mode/src/runtime/callbacks.rs | 26 +++ codex-rs/code-mode/src/runtime/globals.rs | 3 + codex-rs/code-mode/src/runtime/value.rs | 214 +++++++++++++++++--- codex-rs/code-mode/src/service.rs | 199 +++++++++++++++++- codex-rs/core/tests/suite/code_mode.rs | 1 + 6 files changed, 418 insertions(+), 26 deletions(-) diff --git a/codex-rs/code-mode/src/description.rs b/codex-rs/code-mode/src/description.rs index 4c5eb6fbdc..c902e50280 100644 --- a/codex-rs/code-mode/src/description.rs +++ b/codex-rs/code-mode/src/description.rs @@ -27,6 +27,7 @@ const EXEC_DESCRIPTION_TEMPLATE: &str = r#"Run JavaScript code to orchestrate/co - `exit()`: Immediately ends the current script successfully (like an early return from the top level). - `text(value: string | number | boolean | undefined | null)`: Appends a text item. Non-string values are stringified with `JSON.stringify(...)` when possible. - `image(imageUrlOrItem: string | { image_url: string; detail?: "auto" | "low" | "high" | "original" | null } | ImageContent, detail?: "auto" | "low" | "high" | "original" | null)`: Appends an image item. `image_url` can be an HTTPS URL or a base64-encoded `data:` URL. To forward an MCP tool image, pass an individual `ImageContent` block from `result.content`, for example `image(result.content[0])`. MCP image blocks may request detail with `_meta: { "codex/imageDetail": "original" }`. When provided, the second `detail` argument overrides any detail embedded in the first argument. +- `forward_output(...toolOutputs: unknown[])`: Forwards direct tool output values. It loops over MCP `CallToolResult.content` and function-output `content_items`, appending text blocks with `text(...)` and image blocks with `image(...)`. Single image-url result objects are appended as images. Throws for arrays, arbitrary objects, or content block types other than text/image; use `text(...)` or `image(...)` explicitly for those cases. - `store(key: string, value: any)`: stores a serializable value under a string key for later `exec` calls in the same session. - `load(key: string)`: returns the stored value for a string key, or `undefined` if it is missing. - `notify(value: string | number | boolean | undefined | null)`: immediately injects an extra `custom_tool_call_output` for the current `exec` call. Values are stringified like `text(...)`. diff --git a/codex-rs/code-mode/src/runtime/callbacks.rs b/codex-rs/code-mode/src/runtime/callbacks.rs index a9755f6eb0..566c2a7d0f 100644 --- a/codex-rs/code-mode/src/runtime/callbacks.rs +++ b/codex-rs/code-mode/src/runtime/callbacks.rs @@ -5,6 +5,7 @@ use super::RuntimeEvent; use super::RuntimeState; use super::timers; use super::value::json_to_v8; +use super::value::normalize_forward_output_items; use super::value::normalize_output_image; use super::value::serialize_output_text; use super::value::throw_type_error; @@ -132,6 +133,31 @@ pub(super) fn image_callback( retval.set(v8::undefined(scope).into()); } +pub(super) fn forward_output_callback( + scope: &mut v8::PinScope<'_, '_>, + args: v8::FunctionCallbackArguments, + mut retval: v8::ReturnValue, +) { + let mut content_items = Vec::new(); + for index in 0..args.length() { + let mut items = match normalize_forward_output_items(scope, args.get(index)) { + Ok(items) => items, + Err(error_text) => { + throw_type_error(scope, &error_text); + return; + } + }; + content_items.append(&mut items); + } + + if let Some(state) = scope.get_slot::() { + for item in content_items { + let _ = state.event_tx.send(RuntimeEvent::ContentItem(item)); + } + } + retval.set(v8::undefined(scope).into()); +} + pub(super) fn store_callback( scope: &mut v8::PinScope<'_, '_>, args: v8::FunctionCallbackArguments, diff --git a/codex-rs/code-mode/src/runtime/globals.rs b/codex-rs/code-mode/src/runtime/globals.rs index 2ec6953f09..e7ca8e2cfb 100644 --- a/codex-rs/code-mode/src/runtime/globals.rs +++ b/codex-rs/code-mode/src/runtime/globals.rs @@ -1,6 +1,7 @@ use super::RuntimeState; use super::callbacks::clear_timeout_callback; use super::callbacks::exit_callback; +use super::callbacks::forward_output_callback; use super::callbacks::image_callback; use super::callbacks::load_callback; use super::callbacks::notify_callback; @@ -23,6 +24,7 @@ pub(super) fn install_globals(scope: &mut v8::PinScope<'_, '_>) -> Result<(), St let set_timeout = helper_function(scope, "setTimeout", set_timeout_callback)?; let text = helper_function(scope, "text", text_callback)?; let image = helper_function(scope, "image", image_callback)?; + let forward_output = helper_function(scope, "forward_output", forward_output_callback)?; let store = helper_function(scope, "store", store_callback)?; let load = helper_function(scope, "load", load_callback)?; let notify = helper_function(scope, "notify", notify_callback)?; @@ -35,6 +37,7 @@ pub(super) fn install_globals(scope: &mut v8::PinScope<'_, '_>) -> Result<(), St set_global(scope, global, "setTimeout", set_timeout.into())?; set_global(scope, global, "text", text.into())?; set_global(scope, global, "image", image.into())?; + set_global(scope, global, "forward_output", forward_output.into())?; set_global(scope, global, "store", store.into())?; set_global(scope, global, "load", load.into())?; set_global(scope, global, "notify", notify.into())?; diff --git a/codex-rs/code-mode/src/runtime/value.rs b/codex-rs/code-mode/src/runtime/value.rs index 8d76a832d3..0bbe9ed3f6 100644 --- a/codex-rs/code-mode/src/runtime/value.rs +++ b/codex-rs/code-mode/src/runtime/value.rs @@ -6,6 +6,7 @@ use crate::response::ImageDetail; const IMAGE_HELPER_EXPECTS_MESSAGE: &str = "image expects a non-empty image URL string, an object with image_url and optional detail, or a raw MCP image block"; const CODEX_IMAGE_DETAIL_META_KEY: &str = "codex/imageDetail"; +const FORWARD_OUTPUT_HELPER_EXPECTS_MESSAGE: &str = "forward_output expects a direct tool output with `content` or `content_items`, a text/image content item, an image_url object, or a string output"; pub(super) fn serialize_output_text( scope: &mut v8::PinScope<'_, '_>, @@ -55,33 +56,11 @@ pub(super) fn normalize_output_image( return Err(IMAGE_HELPER_EXPECTS_MESSAGE.to_string()); }; - if image_url.is_empty() { - return Err(IMAGE_HELPER_EXPECTS_MESSAGE.to_string()); - } - let lower = image_url.to_ascii_lowercase(); - if !(lower.starts_with("http://") - || lower.starts_with("https://") - || lower.starts_with("data:")) - { - return Err("image expects an http(s) or data URL".to_string()); - } + validate_image_url(&image_url)?; let detail = detail_override.or(detail); let detail = match detail { - Some(detail) => { - let normalized = detail.to_ascii_lowercase(); - Some(match normalized.as_str() { - "auto" => ImageDetail::Auto, - "low" => ImageDetail::Low, - "high" => ImageDetail::High, - "original" => ImageDetail::Original, - _ => { - return Err( - "image detail must be one of: auto, low, high, original".to_string() - ); - } - }) - } + Some(detail) => Some(parse_image_detail(&detail)?), None => Some(DEFAULT_IMAGE_DETAIL), }; @@ -97,6 +76,193 @@ pub(super) fn normalize_output_image( } } +pub(super) fn normalize_forward_output_items( + scope: &mut v8::PinScope<'_, '_>, + value: v8::Local<'_, v8::Value>, +) -> Result, String> { + if value.is_string() { + return Ok(vec![FunctionCallOutputContentItem::InputText { + text: serialize_output_text(scope, value)?, + }]); + } + + let Some(json) = v8_value_to_json(scope, value)? else { + return Err(FORWARD_OUTPUT_HELPER_EXPECTS_MESSAGE.to_string()); + }; + + if let JsonValue::Object(object) = &json + && let Some(items) = forward_output_items_from_tool_output_object(object)? + { + return Ok(items); + } + + Err(FORWARD_OUTPUT_HELPER_EXPECTS_MESSAGE.to_string()) +} + +fn forward_output_items_from_tool_output_object( + object: &serde_json::Map, +) -> Result>, String> { + if let Some(content) = object.get("content") { + let content = content + .as_array() + .ok_or_else(|| "output expected `content` to be an array".to_string())?; + return content_items_from_array(content).map(Some); + } + + if let Some(content_items) = object + .get("content_items") + .or_else(|| object.get("contentItems")) + { + let content_items = content_items + .as_array() + .ok_or_else(|| "output expected `content_items` to be an array".to_string())?; + return content_items_from_array(content_items).map(Some); + } + + if let Some(item) = forward_output_content_item_from_object(object)? { + return Ok(Some(vec![item])); + } + + if let Some(output) = object.get("output").and_then(JsonValue::as_str) { + return Ok(Some(vec![FunctionCallOutputContentItem::InputText { + text: output.to_string(), + }])); + } + + Ok(None) +} + +fn content_items_from_array( + values: &[JsonValue], +) -> Result, String> { + values + .iter() + .map(|value| { + if let JsonValue::Object(object) = value { + return forward_output_content_item_from_object(object)? + .ok_or_else(|| FORWARD_OUTPUT_HELPER_EXPECTS_MESSAGE.to_string()); + } + Err("forward_output expected content entries to be text/image objects".to_string()) + }) + .collect() +} + +fn forward_output_content_item_from_object( + object: &serde_json::Map, +) -> Result, String> { + if let Some(item_type) = object.get("type").and_then(JsonValue::as_str) { + return match item_type { + "text" | "input_text" | "inputText" => { + Ok(Some(FunctionCallOutputContentItem::InputText { + text: required_string_field(object, "text", item_type)?.to_string(), + })) + } + "image" => mcp_image_content_item_from_object(object).map(Some), + "input_image" | "inputImage" => image_url_content_item_from_object(object).map(Some), + _ => Err(format!( + "forward_output only supports text and image content blocks, got `{item_type}`" + )), + }; + } + + if object.contains_key("image_url") || object.contains_key("imageUrl") { + return image_url_content_item_from_object(object).map(Some); + } + + Ok(None) +} + +fn required_string_field<'a>( + object: &'a serde_json::Map, + field: &str, + item_type: &str, +) -> Result<&'a str, String> { + object + .get(field) + .and_then(JsonValue::as_str) + .ok_or_else(|| format!("output expected `{item_type}` content to include `{field}`")) +} + +fn image_url_content_item_from_object( + object: &serde_json::Map, +) -> Result { + let Some(image_url) = object.get("image_url").or_else(|| object.get("imageUrl")) else { + return Err("output expected image content to include `image_url`".to_string()); + }; + let image_url = image_url + .as_str() + .ok_or_else(|| "output expected `image_url` to be a string".to_string())?; + validate_image_url(image_url)?; + let detail = json_image_detail_value(object.get("detail"))?.or(Some(DEFAULT_IMAGE_DETAIL)); + Ok(FunctionCallOutputContentItem::InputImage { + image_url: image_url.to_string(), + detail, + }) +} + +fn mcp_image_content_item_from_object( + object: &serde_json::Map, +) -> Result { + let data = required_string_field(object, "data", "image")?; + if data.is_empty() { + return Err("output expected MCP image data".to_string()); + } + + let image_url = if data.to_ascii_lowercase().starts_with("data:") { + data.to_string() + } else { + let mime_type = object + .get("mimeType") + .or_else(|| object.get("mime_type")) + .and_then(JsonValue::as_str) + .filter(|mime_type| !mime_type.is_empty()) + .unwrap_or("application/octet-stream"); + format!("data:{mime_type};base64,{data}") + }; + validate_image_url(&image_url)?; + + let detail = object + .get("_meta") + .and_then(JsonValue::as_object) + .and_then(|meta| meta.get(CODEX_IMAGE_DETAIL_META_KEY)) + .and_then(JsonValue::as_str) + .and_then(|detail| parse_image_detail(detail).ok()) + .or(Some(DEFAULT_IMAGE_DETAIL)); + + Ok(FunctionCallOutputContentItem::InputImage { image_url, detail }) +} + +fn json_image_detail_value(value: Option<&JsonValue>) -> Result, String> { + match value { + Some(JsonValue::String(detail)) => parse_image_detail(detail).map(Some), + Some(JsonValue::Null) | None => Ok(None), + Some(_) => Err("image detail must be a string when provided".to_string()), + } +} + +fn parse_image_detail(detail: &str) -> Result { + let normalized = detail.to_ascii_lowercase(); + match normalized.as_str() { + "auto" => Ok(ImageDetail::Auto), + "low" => Ok(ImageDetail::Low), + "high" => Ok(ImageDetail::High), + "original" => Ok(ImageDetail::Original), + _ => Err("image detail must be one of: auto, low, high, original".to_string()), + } +} + +fn validate_image_url(image_url: &str) -> Result<(), String> { + if image_url.is_empty() { + return Err(IMAGE_HELPER_EXPECTS_MESSAGE.to_string()); + } + let lower = image_url.to_ascii_lowercase(); + if lower.starts_with("http://") || lower.starts_with("https://") || lower.starts_with("data:") { + Ok(()) + } else { + Err("image expects an http(s) or data URL".to_string()) + } +} + fn parse_non_mcp_output_image( scope: &mut v8::PinScope<'_, '_>, object: v8::Local<'_, v8::Object>, diff --git a/codex-rs/code-mode/src/service.rs b/codex-rs/code-mode/src/service.rs index 7326c834e2..73fdefc483 100644 --- a/codex-rs/code-mode/src/service.rs +++ b/codex-rs/code-mode/src/service.rs @@ -671,7 +671,7 @@ text(formatter.format(new Date("2025-01-02T03:04:05Z"))); } #[tokio::test] - async fn output_helpers_return_undefined() { + async fn global_helpers_return_undefined() { let service = CodeModeService::new(); let response = service @@ -681,6 +681,7 @@ const returnsUndefined = [ text("first"), image("https://example.com/image.jpg"), notify("ping"), + forward_output(), ].map((value) => value === undefined); text(JSON.stringify(returnsUndefined)); "# @@ -704,7 +705,7 @@ text(JSON.stringify(returnsUndefined)); detail: Some(crate::DEFAULT_IMAGE_DETAIL), }, FunctionCallOutputContentItem::InputText { - text: "[true,true,true]".to_string(), + text: "[true,true,true,true]".to_string(), }, ], stored_values: HashMap::new(), @@ -713,6 +714,200 @@ text(JSON.stringify(returnsUndefined)); ); } + #[tokio::test] + async fn forward_output_helper_forwards_mcp_text_and_image_content() { + let service = CodeModeService::new(); + + let response = service + .execute(ExecuteRequest { + source: r#" +forward_output({ + content: [ + { type: "text", text: "alpha" }, + { + type: "image", + data: "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/iZk9HQAAAABJRU5ErkJggg==", + mimeType: "image/png", + _meta: { "codex/imageDetail": "original" }, + }, + { type: "text", text: "omega" }, + ], + isError: false, +}); +"# + .to_string(), + yield_time_ms: None, + ..execute_request("") + }) + .await + .unwrap(); + + assert_eq!( + response, + RuntimeResponse::Result { + cell_id: "1".to_string(), + content_items: vec![ + FunctionCallOutputContentItem::InputText { + text: "alpha".to_string(), + }, + FunctionCallOutputContentItem::InputImage { + image_url: "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/iZk9HQAAAABJRU5ErkJggg==".to_string(), + detail: Some(crate::ImageDetail::Original), + }, + FunctionCallOutputContentItem::InputText { + text: "omega".to_string(), + }, + ], + stored_values: HashMap::new(), + error_text: None, + } + ); + } + + #[tokio::test] + async fn forward_output_helper_allows_output_local_variable() { + let service = CodeModeService::new(); + + let response = service + .execute(ExecuteRequest { + source: r#" +const output = { content: [{ type: "text", text: "ok" }] }; +forward_output(output); +"# + .to_string(), + yield_time_ms: None, + ..execute_request("") + }) + .await + .unwrap(); + + assert_eq!( + response, + RuntimeResponse::Result { + cell_id: "1".to_string(), + content_items: vec![FunctionCallOutputContentItem::InputText { + text: "ok".to_string(), + }], + stored_values: HashMap::new(), + error_text: None, + } + ); + } + + #[tokio::test] + async fn forward_output_helper_forwards_function_items_image_url_objects_and_string_output() { + let service = CodeModeService::new(); + + let response = service + .execute(ExecuteRequest { + source: r#" +forward_output( + { + content_items: [ + { type: "input_text", text: "line 1" }, + { type: "input_image", image_url: "https://example.com/one.jpg", detail: "low" }, + ], + }, + { image_url: "https://example.com/two.jpg", detail: "auto" }, + { output: "command output" }, +); +"# + .to_string(), + yield_time_ms: None, + ..execute_request("") + }) + .await + .unwrap(); + + assert_eq!( + response, + RuntimeResponse::Result { + cell_id: "1".to_string(), + content_items: vec![ + FunctionCallOutputContentItem::InputText { + text: "line 1".to_string(), + }, + FunctionCallOutputContentItem::InputImage { + image_url: "https://example.com/one.jpg".to_string(), + detail: Some(crate::ImageDetail::Low), + }, + FunctionCallOutputContentItem::InputImage { + image_url: "https://example.com/two.jpg".to_string(), + detail: Some(crate::ImageDetail::Auto), + }, + FunctionCallOutputContentItem::InputText { + text: "command output".to_string(), + }, + ], + stored_values: HashMap::new(), + error_text: None, + } + ); + } + + #[tokio::test] + async fn forward_output_helper_rejects_arrays_outside_tool_output_fields() { + let service = CodeModeService::new(); + + let response = service + .execute(ExecuteRequest { + source: r#" +forward_output([{ type: "text", text: "not forwarded directly" }]); +"# + .to_string(), + yield_time_ms: None, + ..execute_request("") + }) + .await + .unwrap(); + + assert_eq!( + response, + RuntimeResponse::Result { + cell_id: "1".to_string(), + content_items: Vec::new(), + stored_values: HashMap::new(), + error_text: Some( + "forward_output expects a direct tool output with `content` or `content_items`, a text/image content item, an image_url object, or a string output".to_string(), + ), + } + ); + } + + #[tokio::test] + async fn forward_output_helper_rejects_unknown_content_types() { + let service = CodeModeService::new(); + + let response = service + .execute(ExecuteRequest { + source: r#" +forward_output({ + content: [ + { type: "audio", data: "AAA", mimeType: "audio/wav" }, + ], +}); +"# + .to_string(), + yield_time_ms: None, + ..execute_request("") + }) + .await + .unwrap(); + + assert_eq!( + response, + RuntimeResponse::Result { + cell_id: "1".to_string(), + content_items: Vec::new(), + stored_values: HashMap::new(), + error_text: Some( + "forward_output only supports text and image content blocks, got `audio`" + .to_string(), + ), + } + ); + } + #[tokio::test] async fn image_helper_accepts_raw_mcp_image_block_with_original_detail() { let service = CodeModeService::new(); diff --git a/codex-rs/core/tests/suite/code_mode.rs b/codex-rs/core/tests/suite/code_mode.rs index 3bcb37e7b2..2a7a919886 100644 --- a/codex-rs/core/tests/suite/code_mode.rs +++ b/codex-rs/core/tests/suite/code_mode.rs @@ -2434,6 +2434,7 @@ text(JSON.stringify(Object.getOwnPropertyNames(globalThis).sort())); "isNaN", "load", "notify", + "forward_output", "parseFloat", "parseInt", "setTimeout",