From dc4e54d06152b3bb91a4e9989299b094d20ad827 Mon Sep 17 00:00:00 2001 From: rhan-oai Date: Tue, 26 May 2026 16:24:33 -0700 Subject: [PATCH] Restore legacy image detail values (#24644) ## Why Older persisted rollouts can contain `input_image.detail` values of `auto` or `low` from before `ImageDetail` was narrowed to `high`/`original`. Current deserialization rejects those values, which can make resume skip later compacted checkpoints and reconstruct an oversized raw suffix before the next compaction attempt. Confirmed Sentry reports fixed by this compatibility path: - [CODEX-1H3F](https://openai.sentry.io/issues/7500642496/) - [CODEX-1H6N](https://openai.sentry.io/issues/7501025347/) - [CODEX-1JDP](https://openai.sentry.io/issues/7504549065/) - [CODEX-1HW6](https://openai.sentry.io/issues/7503407986/) ## Background [openai/codex#20693](https://github.com/openai/codex/pull/20693) added image-detail plumbing for app-server `UserInput` so input images could explicitly request `detail: original`. The Slack discussion behind that PR was about ScreenSpot / bridge evals where user input images were resized, while tool output images already had MCP/code-mode ways to request image detail. In review, the intended new API surface was narrowed to `high` and `original`: default to `high`, allow `original` when callers need unchanged image handling, and avoid encouraging new `auto` or `low` usage. That policy still makes sense for newly emitted values. The missing compatibility piece is persisted history. Older rollouts can already contain `auto` and `low`, and resume reconstructs typed history by deserializing those rollout records. Rejecting old values at that boundary causes valid compacted checkpoints to be skipped. This PR restores `auto` and `low` as real variants so old records deserialize and round-trip without being rewritten as `high`, while product paths can continue to default to `high` and avoid emitting `auto` for new behavior. ## What changed - Restored `ImageDetail::Auto` and `ImageDetail::Low` as first-class protocol values. - Preserved `auto`/`low` through rollout deserialization, MCP image metadata, code-mode image output, and schema/type generation. - Kept local image byte handling conservative: only `original` switches to original-resolution loading; `auto`/`low`/`high` continue through the resize-to-fit path while retaining their detail value. - Added regression coverage for enum round-tripping and code-mode `low` detail handling. ## Testing - `just write-app-server-schema` - `just test -p codex-protocol` - `just test -p codex-tools` - `just test -p codex-code-mode` - `just test -p codex-app-server-protocol` - `just test -p codex-core suite::rmcp_client::stdio_image_responses_preserve_original_detail_metadata` - `just test -p codex-core suite::code_mode::code_mode_can_use_mcp_image_result_with_image_helper` - Loaded broken rollouts on local fixed builds, and started/completed new turns. I also attempted `just test -p codex-core`; the local broad run did not finish green: 2559 tests run, 2467 passed, 55 flaky, 91 failed, 1 timed out. The failures were broad timeout/deadline failures across unrelated areas; targeted changed-path core tests above passed. --- .../schema/json/ClientRequest.json | 2 + .../schema/json/ServerNotification.json | 2 + .../codex_app_server_protocol.schemas.json | 2 + .../codex_app_server_protocol.v2.schemas.json | 2 + .../json/v2/ItemCompletedNotification.json | 2 + .../json/v2/ItemStartedNotification.json | 2 + .../RawResponseItemCompletedNotification.json | 2 + .../schema/json/v2/ReviewStartResponse.json | 2 + .../schema/json/v2/ThreadForkResponse.json | 2 + .../schema/json/v2/ThreadListResponse.json | 2 + .../json/v2/ThreadMetadataUpdateResponse.json | 2 + .../schema/json/v2/ThreadReadResponse.json | 2 + .../schema/json/v2/ThreadResumeParams.json | 2 + .../schema/json/v2/ThreadResumeResponse.json | 2 + .../json/v2/ThreadRollbackResponse.json | 2 + .../schema/json/v2/ThreadStartResponse.json | 2 + .../json/v2/ThreadStartedNotification.json | 2 + .../json/v2/ThreadUnarchiveResponse.json | 2 + .../json/v2/TurnCompletedNotification.json | 2 + .../schema/json/v2/TurnStartParams.json | 2 + .../schema/json/v2/TurnStartResponse.json | 2 + .../json/v2/TurnStartedNotification.json | 2 + .../schema/json/v2/TurnSteerParams.json | 2 + .../schema/typescript/ImageDetail.ts | 2 +- codex-rs/code-mode/src/description.rs | 2 +- codex-rs/code-mode/src/response.rs | 2 + codex-rs/code-mode/src/runtime/value.rs | 8 +++- codex-rs/code-mode/src/service.rs | 38 ++++++++++++++++++- .../src/tools/code_mode/response_adapter.rs | 2 + codex-rs/protocol/src/models.rs | 36 +++++++++++++++++- codex-rs/tools/src/image_detail.rs | 2 +- codex-rs/tools/src/image_detail_tests.rs | 12 +++++- 32 files changed, 140 insertions(+), 10 deletions(-) diff --git a/codex-rs/app-server-protocol/schema/json/ClientRequest.json b/codex-rs/app-server-protocol/schema/json/ClientRequest.json index 552defad1b..63e8fbde1a 100644 --- a/codex-rs/app-server-protocol/schema/json/ClientRequest.json +++ b/codex-rs/app-server-protocol/schema/json/ClientRequest.json @@ -1085,6 +1085,8 @@ }, "ImageDetail": { "enum": [ + "auto", + "low", "high", "original" ], diff --git a/codex-rs/app-server-protocol/schema/json/ServerNotification.json b/codex-rs/app-server-protocol/schema/json/ServerNotification.json index 90899cb152..2c290558da 100644 --- a/codex-rs/app-server-protocol/schema/json/ServerNotification.json +++ b/codex-rs/app-server-protocol/schema/json/ServerNotification.json @@ -2031,6 +2031,8 @@ }, "ImageDetail": { "enum": [ + "auto", + "low", "high", "original" ], diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json index ab19f3e770..28dc006b5e 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json @@ -10160,6 +10160,8 @@ }, "ImageDetail": { "enum": [ + "auto", + "low", "high", "original" ], diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json index b12b0eedd8..d1c59e9855 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json @@ -6640,6 +6640,8 @@ }, "ImageDetail": { "enum": [ + "auto", + "low", "high", "original" ], diff --git a/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json index 87d36d2927..cea3dae7d2 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json @@ -287,6 +287,8 @@ }, "ImageDetail": { "enum": [ + "auto", + "low", "high", "original" ], diff --git a/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json index d45fddf4ba..529667388e 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json @@ -287,6 +287,8 @@ }, "ImageDetail": { "enum": [ + "auto", + "low", "high", "original" ], diff --git a/codex-rs/app-server-protocol/schema/json/v2/RawResponseItemCompletedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/RawResponseItemCompletedNotification.json index e69b342282..f82acbae64 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/RawResponseItemCompletedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/RawResponseItemCompletedNotification.json @@ -165,6 +165,8 @@ }, "ImageDetail": { "enum": [ + "auto", + "low", "high", "original" ], diff --git a/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json index 877eeb1278..df93fbbea5 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json @@ -424,6 +424,8 @@ }, "ImageDetail": { "enum": [ + "auto", + "low", "high", "original" ], diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json index 8c45f57d38..a9a2529fc5 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json @@ -529,6 +529,8 @@ }, "ImageDetail": { "enum": [ + "auto", + "low", "high", "original" ], diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json index a9a186228b..8ccceaaa7c 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json @@ -450,6 +450,8 @@ }, "ImageDetail": { "enum": [ + "auto", + "low", "high", "original" ], diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json index a2a4834cb3..87af03ea62 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json @@ -450,6 +450,8 @@ }, "ImageDetail": { "enum": [ + "auto", + "low", "high", "original" ], diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json index 5d76f9b181..a6d86c3485 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json @@ -450,6 +450,8 @@ }, "ImageDetail": { "enum": [ + "auto", + "low", "high", "original" ], diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeParams.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeParams.json index d4bdeda0ed..58b85d41a4 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeParams.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeParams.json @@ -224,6 +224,8 @@ }, "ImageDetail": { "enum": [ + "auto", + "low", "high", "original" ], diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json index 8055ff58fc..b4fa181c88 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json @@ -529,6 +529,8 @@ }, "ImageDetail": { "enum": [ + "auto", + "low", "high", "original" ], diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadRollbackResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadRollbackResponse.json index d0e25a1eaa..4256d13ec4 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadRollbackResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadRollbackResponse.json @@ -450,6 +450,8 @@ }, "ImageDetail": { "enum": [ + "auto", + "low", "high", "original" ], diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json index d456ccebc4..6ab655a8bc 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json @@ -529,6 +529,8 @@ }, "ImageDetail": { "enum": [ + "auto", + "low", "high", "original" ], diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json index dd7de34b82..91bc13a9c6 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json @@ -450,6 +450,8 @@ }, "ImageDetail": { "enum": [ + "auto", + "low", "high", "original" ], diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json index 5929510520..0b68bfa957 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json @@ -450,6 +450,8 @@ }, "ImageDetail": { "enum": [ + "auto", + "low", "high", "original" ], diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json index ddaab863f7..4856f73529 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json @@ -424,6 +424,8 @@ }, "ImageDetail": { "enum": [ + "auto", + "low", "high", "original" ], diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnStartParams.json b/codex-rs/app-server-protocol/schema/json/v2/TurnStartParams.json index ffc130dcc9..17b69a698e 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/TurnStartParams.json +++ b/codex-rs/app-server-protocol/schema/json/v2/TurnStartParams.json @@ -123,6 +123,8 @@ }, "ImageDetail": { "enum": [ + "auto", + "low", "high", "original" ], diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json b/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json index a2836f7f03..c5e48ae6ce 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json @@ -424,6 +424,8 @@ }, "ImageDetail": { "enum": [ + "auto", + "low", "high", "original" ], diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json index 8c57890a3b..075699a34e 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json @@ -424,6 +424,8 @@ }, "ImageDetail": { "enum": [ + "auto", + "low", "high", "original" ], diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnSteerParams.json b/codex-rs/app-server-protocol/schema/json/v2/TurnSteerParams.json index ef05b27673..6f5452de66 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/TurnSteerParams.json +++ b/codex-rs/app-server-protocol/schema/json/v2/TurnSteerParams.json @@ -44,6 +44,8 @@ }, "ImageDetail": { "enum": [ + "auto", + "low", "high", "original" ], diff --git a/codex-rs/app-server-protocol/schema/typescript/ImageDetail.ts b/codex-rs/app-server-protocol/schema/typescript/ImageDetail.ts index 5a62cc32f1..a48f07c088 100644 --- a/codex-rs/app-server-protocol/schema/typescript/ImageDetail.ts +++ b/codex-rs/app-server-protocol/schema/typescript/ImageDetail.ts @@ -2,4 +2,4 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. -export type ImageDetail = "high" | "original"; +export type ImageDetail = "auto" | "low" | "high" | "original"; diff --git a/codex-rs/code-mode/src/description.rs b/codex-rs/code-mode/src/description.rs index 18d3c4555c..0c2813e51a 100644 --- a/codex-rs/code-mode/src/description.rs +++ b/codex-rs/code-mode/src/description.rs @@ -24,7 +24,7 @@ const EXEC_DESCRIPTION_TEMPLATE: &str = r#"Run JavaScript code to orchestrate/co - Global helpers: - `exit()`: Immediately ends the current script successfully (like an early return from the top level). - `text(value: string | number | boolean | undefined | null)`: Appends a text item. Non-string values are stringified with `JSON.stringify(...)` when possible. -- `image(imageUrlOrItem: string | { image_url: string; detail?: "high" | "original" | null } | ImageContent, detail?: "high" | "original" | null)`: Appends an image item. `image_url` can be an HTTPS URL or a base64-encoded `data:` URL. To forward an MCP tool image, pass an individual `ImageContent` block from `result.content`, for example `image(result.content[0])`. MCP image blocks may request detail with `_meta: { "codex/imageDetail": "original" }`. When provided, the second `detail` argument overrides any detail embedded in the first argument. +- `image(imageUrlOrItem: string | { image_url: string; detail?: "auto" | "low" | "high" | "original" | null } | ImageContent, detail?: "auto" | "low" | "high" | "original" | null)`: Appends an image item. `image_url` can be an HTTPS URL or a base64-encoded `data:` URL. To forward an MCP tool image, pass an individual `ImageContent` block from `result.content`, for example `image(result.content[0])`. MCP image blocks may request detail with `_meta: { "codex/imageDetail": "original" }`. When provided, the second `detail` argument overrides any detail embedded in the first argument. - `store(key: string, value: any)`: stores a serializable value under a string key for later `exec` calls in the same session. - `load(key: string)`: returns the stored value for a string key, or `undefined` if it is missing. - `notify(value: string | number | boolean | undefined | null)`: immediately injects an extra `custom_tool_call_output` for the current `exec` call. Values are stringified like `text(...)`. diff --git a/codex-rs/code-mode/src/response.rs b/codex-rs/code-mode/src/response.rs index ae92639cc0..0ac3a03770 100644 --- a/codex-rs/code-mode/src/response.rs +++ b/codex-rs/code-mode/src/response.rs @@ -4,6 +4,8 @@ use serde::Serialize; #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] #[serde(rename_all = "lowercase")] pub enum ImageDetail { + Auto, + Low, High, Original, } diff --git a/codex-rs/code-mode/src/runtime/value.rs b/codex-rs/code-mode/src/runtime/value.rs index 865b5a569d..8d76a832d3 100644 --- a/codex-rs/code-mode/src/runtime/value.rs +++ b/codex-rs/code-mode/src/runtime/value.rs @@ -71,10 +71,14 @@ pub(super) fn normalize_output_image( Some(detail) => { let normalized = detail.to_ascii_lowercase(); Some(match normalized.as_str() { + "auto" => ImageDetail::Auto, + "low" => ImageDetail::Low, "high" => ImageDetail::High, "original" => ImageDetail::Original, _ => { - return Err("image detail must be one of: high, original".to_string()); + return Err( + "image detail must be one of: auto, low, high, original".to_string() + ); } }) } @@ -156,7 +160,7 @@ fn parse_mcp_output_image( .and_then(JsonValue::as_object) .and_then(|meta| meta.get(CODEX_IMAGE_DETAIL_META_KEY)) .and_then(JsonValue::as_str) - .filter(|detail| matches!(*detail, "high" | "original")) + .filter(|detail| matches!(*detail, "auto" | "low" | "high" | "original")) .map(str::to_string); Ok((image_url, detail)) } diff --git a/codex-rs/code-mode/src/service.rs b/codex-rs/code-mode/src/service.rs index de4ed13e58..d153f1b416 100644 --- a/codex-rs/code-mode/src/service.rs +++ b/codex-rs/code-mode/src/service.rs @@ -1359,7 +1359,7 @@ image( } #[tokio::test] - async fn image_helper_rejects_unsupported_detail() { + async fn image_helper_accepts_low_detail() { let service = CodeModeService::new(); let response = service @@ -1369,6 +1369,38 @@ image({ image_url: "https://example.com/image.jpg", detail: "low", }); +"# + .to_string(), + yield_time_ms: None, + ..execute_request("") + }) + .await + .unwrap(); + + assert_eq!( + response, + RuntimeResponse::Result { + cell_id: "1".to_string(), + content_items: vec![FunctionCallOutputContentItem::InputImage { + image_url: "https://example.com/image.jpg".to_string(), + detail: Some(crate::ImageDetail::Low), + }], + error_text: None, + } + ); + } + + #[tokio::test] + async fn image_helper_rejects_unsupported_detail() { + let service = CodeModeService::new(); + + let response = service + .execute(ExecuteRequest { + source: r#" +image({ + image_url: "https://example.com/image.jpg", + detail: "medium", +}); "# .to_string(), yield_time_ms: None, @@ -1382,7 +1414,9 @@ image({ RuntimeResponse::Result { cell_id: "1".to_string(), content_items: Vec::new(), - error_text: Some("image detail must be one of: high, original".to_string()), + error_text: Some( + "image detail must be one of: auto, low, high, original".to_string() + ), } ); } diff --git a/codex-rs/core/src/tools/code_mode/response_adapter.rs b/codex-rs/core/src/tools/code_mode/response_adapter.rs index 133d6e2fc1..e20cf6a071 100644 --- a/codex-rs/core/src/tools/code_mode/response_adapter.rs +++ b/codex-rs/core/src/tools/code_mode/response_adapter.rs @@ -17,6 +17,8 @@ impl IntoProtocol for CodeModeImageDetail { fn into_protocol(self) -> ImageDetail { let value = self; match value { + CodeModeImageDetail::Auto => ImageDetail::Auto, + CodeModeImageDetail::Low => ImageDetail::Low, CodeModeImageDetail::High => ImageDetail::High, CodeModeImageDetail::Original => ImageDetail::Original, } diff --git a/codex-rs/protocol/src/models.rs b/codex-rs/protocol/src/models.rs index 85c5a5d704..86603db1ef 100644 --- a/codex-rs/protocol/src/models.rs +++ b/codex-rs/protocol/src/models.rs @@ -725,6 +725,8 @@ pub enum ContentItem { #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, JsonSchema, TS)] #[serde(rename_all = "lowercase")] pub enum ImageDetail { + Auto, + Low, High, Original, } @@ -1077,7 +1079,7 @@ pub fn local_image_content_items_with_label_number( ) -> Vec { let mode = match detail { ImageDetail::Original => PromptImageMode::Original, - ImageDetail::High => PromptImageMode::ResizeToFit, + ImageDetail::Auto | ImageDetail::Low | ImageDetail::High => PromptImageMode::ResizeToFit, }; match load_for_prompt_bytes(path, file_bytes, mode) { @@ -1599,6 +1601,8 @@ fn convert_mcp_content_to_items( .and_then(|meta| meta.get(CODEX_IMAGE_DETAIL_META_KEY)) .and_then(serde_json::Value::as_str) .and_then(|detail| match detail { + "auto" => Some(ImageDetail::Auto), + "low" => Some(ImageDetail::Low), "high" => Some(ImageDetail::High), "original" => Some(ImageDetail::Original), _ => None, @@ -1674,6 +1678,36 @@ mod tests { ); } + #[test] + fn image_detail_roundtrips_all_wire_values() -> Result<()> { + assert_eq!( + serde_json::from_str::("\"auto\"")?, + ImageDetail::Auto + ); + assert_eq!( + serde_json::from_str::("\"low\"")?, + ImageDetail::Low + ); + assert_eq!(serde_json::to_string(&ImageDetail::Auto)?, "\"auto\""); + assert_eq!(serde_json::to_string(&ImageDetail::Low)?, "\"low\""); + + let content_item: ContentItem = serde_json::from_value(serde_json::json!({ + "type": "input_image", + "image_url": "data:image/png;base64,abc", + "detail": "auto", + }))?; + + assert_eq!( + content_item, + ContentItem::InputImage { + image_url: "data:image/png;base64,abc".to_string(), + detail: Some(ImageDetail::Auto), + } + ); + + Ok(()) + } + #[test] fn sandbox_permissions_helpers_match_documented_semantics() { let cases = [ diff --git a/codex-rs/tools/src/image_detail.rs b/codex-rs/tools/src/image_detail.rs index 145cda663c..37086f691d 100644 --- a/codex-rs/tools/src/image_detail.rs +++ b/codex-rs/tools/src/image_detail.rs @@ -16,7 +16,7 @@ pub fn normalize_output_image_detail( Some(ImageDetail::Original) } Some(ImageDetail::Original) | None => None, - Some(ImageDetail::High) => Some(ImageDetail::High), + Some(ImageDetail::Auto | ImageDetail::Low | ImageDetail::High) => detail, } } diff --git a/codex-rs/tools/src/image_detail_tests.rs b/codex-rs/tools/src/image_detail_tests.rs index 919537acf7..2e46b49d08 100644 --- a/codex-rs/tools/src/image_detail_tests.rs +++ b/codex-rs/tools/src/image_detail_tests.rs @@ -70,6 +70,14 @@ fn explicit_original_is_dropped_without_model_support() { fn explicit_non_original_detail_is_preserved() { let model_info = model_info(); + assert_eq!( + normalize_output_image_detail(&model_info, Some(ImageDetail::Auto)), + Some(ImageDetail::Auto) + ); + assert_eq!( + normalize_output_image_detail(&model_info, Some(ImageDetail::Low)), + Some(ImageDetail::Low) + ); assert_eq!( normalize_output_image_detail(&model_info, Some(ImageDetail::High)), Some(ImageDetail::High) @@ -88,7 +96,7 @@ fn sanitize_original_falls_back_to_high_without_support() { }, FunctionCallOutputContentItem::InputImage { image_url: "data:image/png;base64,BBB".to_string(), - detail: Some(ImageDetail::High), + detail: Some(ImageDetail::Low), }, ]; @@ -106,7 +114,7 @@ fn sanitize_original_falls_back_to_high_without_support() { }, FunctionCallOutputContentItem::InputImage { image_url: "data:image/png;base64,BBB".to_string(), - detail: Some(ImageDetail::High), + detail: Some(ImageDetail::Low), }, ] );