mirror of
https://github.com/openai/codex.git
synced 2026-06-01 19:02:59 +00:00
Add under-development original-resolution view_image support (#13050)
## Summary
Add original-resolution support for `view_image` behind the
under-development `view_image_original_resolution` feature flag.
When the flag is enabled and the target model is `gpt-5.3-codex` or
newer, `view_image` now preserves original PNG/JPEG/WebP bytes and sends
`detail: "original"` to the Responses API instead of using the legacy
resize/compress path.
## What changed
- Added `view_image_original_resolution` as an under-development feature
flag.
- Added `ImageDetail` to the protocol models and support for serializing
`detail: "original"` on tool-returned images.
- Added `PromptImageMode::Original` to `codex-utils-image`.
- Preserves original PNG/JPEG/WebP bytes.
- Keeps legacy behavior for the resize path.
- Updated `view_image` to:
- use the shared `local_image_content_items_with_label_number(...)`
helper in both code paths
- select original-resolution mode only when:
- the feature flag is enabled, and
- the model slug parses as `gpt-5.3-codex` or newer
- Kept local user image attachments on the existing resize path; this
change is specific to `view_image`.
- Updated history/image accounting so only `detail: "original"` images
use the docs-based GPT-5 image cost calculation; legacy images still use
the old fixed estimate.
- Added JS REPL guidance, gated on the same feature flag, to prefer JPEG
at 85% quality unless lossless is required, while still allowing other
formats when explicitly requested.
- Updated tests and helper code that construct
`FunctionCallOutputContentItem::InputImage` to carry the new `detail`
field.
## Behavior
### Feature off
- `view_image` keeps the existing resize/re-encode behavior.
- History estimation keeps the existing fixed-cost heuristic.
### Feature on + `gpt-5.3-codex+`
- `view_image` sends original-resolution images with `detail:
"original"`.
- PNG/JPEG/WebP source bytes are preserved when possible.
- History estimation uses the GPT-5 docs-based image-cost calculation
for those `detail: "original"` images.
#### [git stack](https://github.com/magus/git-stack-cli)
- 👉 `1` https://github.com/openai/codex/pull/13050
- ⏳ `2` https://github.com/openai/codex/pull/13331
- ⏳ `3` https://github.com/openai/codex/pull/13049
This commit is contained in:
Curtis 'Fjord' Hawthorne
committed by
GitHub
GitHub
parent
935754baa3
commit
b92146d48b
@@ -1305,6 +1305,7 @@ mod tests {
|
||||
"apply_patch_tool_type": null,
|
||||
"truncation_policy": {"mode": "bytes", "limit": 10000},
|
||||
"supports_parallel_tool_calls": false,
|
||||
"supports_image_detail_original": false,
|
||||
"context_window": 272000,
|
||||
"auto_compact_token_limit": null,
|
||||
"experimental_supported_tools": []
|
||||
|
||||
@@ -2,21 +2,29 @@ use crate::codex::TurnContext;
|
||||
use crate::context_manager::normalize;
|
||||
use crate::event_mapping::is_contextual_user_message_content;
|
||||
use crate::truncate::TruncationPolicy;
|
||||
use crate::truncate::approx_bytes_for_tokens;
|
||||
use crate::truncate::approx_token_count;
|
||||
use crate::truncate::approx_tokens_from_byte_count_i64;
|
||||
use crate::truncate::truncate_function_output_items_with_policy;
|
||||
use crate::truncate::truncate_text;
|
||||
use base64::Engine;
|
||||
use base64::engine::general_purpose::STANDARD as BASE64_STANDARD;
|
||||
use codex_protocol::models::BaseInstructions;
|
||||
use codex_protocol::models::ContentItem;
|
||||
use codex_protocol::models::FunctionCallOutputBody;
|
||||
use codex_protocol::models::FunctionCallOutputContentItem;
|
||||
use codex_protocol::models::FunctionCallOutputPayload;
|
||||
use codex_protocol::models::ImageDetail;
|
||||
use codex_protocol::models::ResponseItem;
|
||||
use codex_protocol::openai_models::InputModality;
|
||||
use codex_protocol::protocol::TokenUsage;
|
||||
use codex_protocol::protocol::TokenUsageInfo;
|
||||
use codex_protocol::protocol::TurnContextItem;
|
||||
use codex_utils_cache::BlockingLruCache;
|
||||
use codex_utils_cache::sha1_digest;
|
||||
use std::num::NonZeroUsize;
|
||||
use std::ops::Deref;
|
||||
use std::sync::LazyLock;
|
||||
|
||||
/// Transcript of thread history
|
||||
#[derive(Debug, Clone, Default)]
|
||||
@@ -428,7 +436,19 @@ fn estimate_item_token_count(item: &ResponseItem) -> i64 {
|
||||
///
|
||||
/// The estimator later converts bytes to tokens using a 4-bytes/token heuristic
|
||||
/// with ceiling division, so 7,373 bytes maps to approximately 1,844 tokens.
|
||||
const IMAGE_BYTES_ESTIMATE: i64 = 7373;
|
||||
const RESIZED_IMAGE_BYTES_ESTIMATE: i64 = 7373;
|
||||
// See https://platform.openai.com/docs/guides/images-vision#calculating-costs.
|
||||
// Use a direct 32px patch count only for `detail: "original"`;
|
||||
// all other image inputs continue to use `RESIZED_IMAGE_BYTES_ESTIMATE`.
|
||||
const ORIGINAL_IMAGE_PATCH_SIZE: u32 = 32;
|
||||
const ORIGINAL_IMAGE_ESTIMATE_CACHE_SIZE: usize = 32;
|
||||
|
||||
static ORIGINAL_IMAGE_ESTIMATE_CACHE: LazyLock<BlockingLruCache<[u8; 20], Option<i64>>> =
|
||||
LazyLock::new(|| {
|
||||
BlockingLruCache::new(
|
||||
NonZeroUsize::new(ORIGINAL_IMAGE_ESTIMATE_CACHE_SIZE).unwrap_or(NonZeroUsize::MIN),
|
||||
)
|
||||
});
|
||||
|
||||
pub(crate) fn estimate_response_item_model_visible_bytes(item: &ResponseItem) -> i64 {
|
||||
match item {
|
||||
@@ -444,15 +464,15 @@ pub(crate) fn estimate_response_item_model_visible_bytes(item: &ResponseItem) ->
|
||||
let raw = serde_json::to_string(item)
|
||||
.map(|serialized| i64::try_from(serialized.len()).unwrap_or(i64::MAX))
|
||||
.unwrap_or_default();
|
||||
let (payload_bytes, image_count) = image_data_url_estimate_adjustment(item);
|
||||
if payload_bytes == 0 || image_count == 0 {
|
||||
let (payload_bytes, replacement_bytes) = image_data_url_estimate_adjustment(item);
|
||||
if payload_bytes == 0 || replacement_bytes == 0 {
|
||||
raw
|
||||
} else {
|
||||
// Replace raw base64 payload bytes with a fixed per-image cost.
|
||||
// We intentionally preserve the data URL prefix and JSON wrapper
|
||||
// bytes already included in `raw`.
|
||||
// Replace raw base64 payload bytes with a per-image estimate.
|
||||
// We intentionally preserve the data URL prefix and JSON
|
||||
// wrapper bytes already included in `raw`.
|
||||
raw.saturating_sub(payload_bytes)
|
||||
.saturating_add(image_count.saturating_mul(IMAGE_BYTES_ESTIMATE))
|
||||
.saturating_add(replacement_bytes)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -463,7 +483,7 @@ pub(crate) fn estimate_response_item_model_visible_bytes(item: &ResponseItem) ->
|
||||
///
|
||||
/// We only discount payloads for `data:image/...;base64,...` URLs (case
|
||||
/// insensitive markers) and leave everything else at raw serialized size.
|
||||
fn base64_data_url_payload_len(url: &str) -> Option<usize> {
|
||||
fn parse_base64_image_data_url(url: &str) -> Option<&str> {
|
||||
if !url
|
||||
.get(.."data:".len())
|
||||
.is_some_and(|prefix| prefix.eq_ignore_ascii_case("data:"))
|
||||
@@ -489,22 +509,62 @@ fn base64_data_url_payload_len(url: &str) -> Option<usize> {
|
||||
if !has_base64_marker {
|
||||
return None;
|
||||
}
|
||||
Some(payload.len())
|
||||
Some(payload)
|
||||
}
|
||||
|
||||
fn estimate_original_image_bytes(image_url: &str) -> Option<i64> {
|
||||
let key = sha1_digest(image_url.as_bytes());
|
||||
ORIGINAL_IMAGE_ESTIMATE_CACHE.get_or_insert_with(key, || {
|
||||
let payload = match parse_base64_image_data_url(image_url) {
|
||||
Some(payload) => payload,
|
||||
None => {
|
||||
tracing::trace!("skipping original-detail estimate for non-base64 image data URL");
|
||||
return None;
|
||||
}
|
||||
};
|
||||
let bytes = match BASE64_STANDARD.decode(payload) {
|
||||
Ok(bytes) => bytes,
|
||||
Err(error) => {
|
||||
tracing::trace!("failed to decode original-detail image payload: {error}");
|
||||
return None;
|
||||
}
|
||||
};
|
||||
let dynamic = match image::load_from_memory(&bytes) {
|
||||
Ok(dynamic) => dynamic,
|
||||
Err(error) => {
|
||||
tracing::trace!("failed to decode original-detail image bytes: {error}");
|
||||
return None;
|
||||
}
|
||||
};
|
||||
let width = i64::from(dynamic.width());
|
||||
let height = i64::from(dynamic.height());
|
||||
let patch_size = i64::from(ORIGINAL_IMAGE_PATCH_SIZE);
|
||||
let patches_wide = width.saturating_add(patch_size.saturating_sub(1)) / patch_size;
|
||||
let patches_high = height.saturating_add(patch_size.saturating_sub(1)) / patch_size;
|
||||
let patch_count = patches_wide.saturating_mul(patches_high);
|
||||
let patch_count = usize::try_from(patch_count).unwrap_or(usize::MAX);
|
||||
Some(i64::try_from(approx_bytes_for_tokens(patch_count)).unwrap_or(i64::MAX))
|
||||
})
|
||||
}
|
||||
|
||||
/// Scans one response item for discount-eligible inline image data URLs and
|
||||
/// returns:
|
||||
/// - total base64 payload bytes to subtract from raw serialized size
|
||||
/// - count of qualifying images to replace with `IMAGE_BYTES_ESTIMATE`
|
||||
/// - total replacement byte estimate for those images
|
||||
fn image_data_url_estimate_adjustment(item: &ResponseItem) -> (i64, i64) {
|
||||
let mut payload_bytes = 0i64;
|
||||
let mut image_count = 0i64;
|
||||
let mut replacement_bytes = 0i64;
|
||||
|
||||
let mut accumulate = |image_url: &str| {
|
||||
if let Some(payload_len) = base64_data_url_payload_len(image_url) {
|
||||
let mut accumulate = |image_url: &str, detail: Option<ImageDetail>| {
|
||||
if let Some(payload_len) = parse_base64_image_data_url(image_url).map(str::len) {
|
||||
payload_bytes =
|
||||
payload_bytes.saturating_add(i64::try_from(payload_len).unwrap_or(i64::MAX));
|
||||
image_count = image_count.saturating_add(1);
|
||||
replacement_bytes = replacement_bytes.saturating_add(match detail {
|
||||
Some(ImageDetail::Original) => {
|
||||
estimate_original_image_bytes(image_url).unwrap_or(RESIZED_IMAGE_BYTES_ESTIMATE)
|
||||
}
|
||||
_ => RESIZED_IMAGE_BYTES_ESTIMATE,
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
@@ -512,7 +572,7 @@ fn image_data_url_estimate_adjustment(item: &ResponseItem) -> (i64, i64) {
|
||||
ResponseItem::Message { content, .. } => {
|
||||
for content_item in content {
|
||||
if let ContentItem::InputImage { image_url } = content_item {
|
||||
accumulate(image_url);
|
||||
accumulate(image_url, None);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -520,8 +580,10 @@ fn image_data_url_estimate_adjustment(item: &ResponseItem) -> (i64, i64) {
|
||||
| ResponseItem::CustomToolCallOutput { output, .. } => {
|
||||
if let FunctionCallOutputBody::ContentItems(items) = &output.body {
|
||||
for content_item in items {
|
||||
if let FunctionCallOutputContentItem::InputImage { image_url } = content_item {
|
||||
accumulate(image_url);
|
||||
if let FunctionCallOutputContentItem::InputImage { image_url, detail } =
|
||||
content_item
|
||||
{
|
||||
accumulate(image_url, *detail);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -529,7 +591,7 @@ fn image_data_url_estimate_adjustment(item: &ResponseItem) -> (i64, i64) {
|
||||
_ => {}
|
||||
}
|
||||
|
||||
(payload_bytes, image_count)
|
||||
(payload_bytes, replacement_bytes)
|
||||
}
|
||||
|
||||
fn is_model_generated_item(item: &ResponseItem) -> bool {
|
||||
|
||||
@@ -1,12 +1,15 @@
|
||||
use super::*;
|
||||
use crate::truncate;
|
||||
use crate::truncate::TruncationPolicy;
|
||||
use base64::Engine;
|
||||
use base64::engine::general_purpose::STANDARD as BASE64_STANDARD;
|
||||
use codex_git::GhostCommit;
|
||||
use codex_protocol::models::BaseInstructions;
|
||||
use codex_protocol::models::ContentItem;
|
||||
use codex_protocol::models::FunctionCallOutputBody;
|
||||
use codex_protocol::models::FunctionCallOutputContentItem;
|
||||
use codex_protocol::models::FunctionCallOutputPayload;
|
||||
use codex_protocol::models::ImageDetail;
|
||||
use codex_protocol::models::LocalShellAction;
|
||||
use codex_protocol::models::LocalShellExecAction;
|
||||
use codex_protocol::models::LocalShellStatus;
|
||||
@@ -14,6 +17,9 @@ use codex_protocol::models::ReasoningItemContent;
|
||||
use codex_protocol::models::ReasoningItemReasoningSummary;
|
||||
use codex_protocol::openai_models::InputModality;
|
||||
use codex_protocol::openai_models::default_input_modalities;
|
||||
use image::ImageBuffer;
|
||||
use image::ImageFormat;
|
||||
use image::Rgba;
|
||||
use pretty_assertions::assert_eq;
|
||||
use regex_lite::Regex;
|
||||
|
||||
@@ -276,6 +282,7 @@ fn for_prompt_strips_images_when_model_does_not_support_images() {
|
||||
},
|
||||
FunctionCallOutputContentItem::InputImage {
|
||||
image_url: "https://example.com/result.png".to_string(),
|
||||
detail: None,
|
||||
},
|
||||
]),
|
||||
},
|
||||
@@ -294,6 +301,7 @@ fn for_prompt_strips_images_when_model_does_not_support_images() {
|
||||
},
|
||||
FunctionCallOutputContentItem::InputImage {
|
||||
image_url: "https://example.com/js-repl-result.png".to_string(),
|
||||
detail: None,
|
||||
},
|
||||
]),
|
||||
},
|
||||
@@ -489,6 +497,7 @@ fn replace_last_turn_images_replaces_tool_output_images() {
|
||||
body: FunctionCallOutputBody::ContentItems(vec![
|
||||
FunctionCallOutputContentItem::InputImage {
|
||||
image_url: "data:image/png;base64,AAA".to_string(),
|
||||
detail: None,
|
||||
},
|
||||
]),
|
||||
success: Some(true),
|
||||
@@ -1302,7 +1311,7 @@ fn image_data_url_payload_does_not_dominate_message_estimate() {
|
||||
|
||||
let raw_len = serde_json::to_string(&image_item).unwrap().len() as i64;
|
||||
let estimated = estimate_response_item_model_visible_bytes(&image_item);
|
||||
let expected = raw_len - payload.len() as i64 + IMAGE_BYTES_ESTIMATE;
|
||||
let expected = raw_len - payload.len() as i64 + RESIZED_IMAGE_BYTES_ESTIMATE;
|
||||
let text_only_estimated = estimate_response_item_model_visible_bytes(&text_only_item);
|
||||
|
||||
assert_eq!(estimated, expected);
|
||||
@@ -1320,13 +1329,16 @@ fn image_data_url_payload_does_not_dominate_function_call_output_estimate() {
|
||||
FunctionCallOutputContentItem::InputText {
|
||||
text: "Screenshot captured".to_string(),
|
||||
},
|
||||
FunctionCallOutputContentItem::InputImage { image_url },
|
||||
FunctionCallOutputContentItem::InputImage {
|
||||
image_url,
|
||||
detail: None,
|
||||
},
|
||||
]),
|
||||
};
|
||||
|
||||
let raw_len = serde_json::to_string(&item).unwrap().len() as i64;
|
||||
let estimated = estimate_response_item_model_visible_bytes(&item);
|
||||
let expected = raw_len - payload.len() as i64 + IMAGE_BYTES_ESTIMATE;
|
||||
let expected = raw_len - payload.len() as i64 + RESIZED_IMAGE_BYTES_ESTIMATE;
|
||||
|
||||
assert_eq!(estimated, expected);
|
||||
assert!(estimated < raw_len);
|
||||
@@ -1342,13 +1354,16 @@ fn image_data_url_payload_does_not_dominate_custom_tool_call_output_estimate() {
|
||||
FunctionCallOutputContentItem::InputText {
|
||||
text: "Screenshot captured".to_string(),
|
||||
},
|
||||
FunctionCallOutputContentItem::InputImage { image_url },
|
||||
FunctionCallOutputContentItem::InputImage {
|
||||
image_url,
|
||||
detail: None,
|
||||
},
|
||||
]),
|
||||
};
|
||||
|
||||
let raw_len = serde_json::to_string(&item).unwrap().len() as i64;
|
||||
let estimated = estimate_response_item_model_visible_bytes(&item);
|
||||
let expected = raw_len - payload.len() as i64 + IMAGE_BYTES_ESTIMATE;
|
||||
let expected = raw_len - payload.len() as i64 + RESIZED_IMAGE_BYTES_ESTIMATE;
|
||||
|
||||
assert_eq!(estimated, expected);
|
||||
assert!(estimated < raw_len);
|
||||
@@ -1370,6 +1385,7 @@ fn non_base64_image_urls_are_unchanged() {
|
||||
output: FunctionCallOutputPayload::from_content_items(vec![
|
||||
FunctionCallOutputContentItem::InputImage {
|
||||
image_url: "file:///tmp/foo.png".to_string(),
|
||||
detail: None,
|
||||
},
|
||||
]),
|
||||
};
|
||||
@@ -1409,7 +1425,10 @@ fn non_image_base64_data_url_is_unchanged() {
|
||||
let item = ResponseItem::FunctionCallOutput {
|
||||
call_id: "call-octet".to_string(),
|
||||
output: FunctionCallOutputPayload::from_content_items(vec![
|
||||
FunctionCallOutputContentItem::InputImage { image_url },
|
||||
FunctionCallOutputContentItem::InputImage {
|
||||
image_url,
|
||||
detail: None,
|
||||
},
|
||||
]),
|
||||
};
|
||||
|
||||
@@ -1433,7 +1452,7 @@ fn mixed_case_data_url_markers_are_adjusted() {
|
||||
|
||||
let raw_len = serde_json::to_string(&item).unwrap().len() as i64;
|
||||
let estimated = estimate_response_item_model_visible_bytes(&item);
|
||||
let expected = raw_len - payload.len() as i64 + IMAGE_BYTES_ESTIMATE;
|
||||
let expected = raw_len - payload.len() as i64 + RESIZED_IMAGE_BYTES_ESTIMATE;
|
||||
|
||||
assert_eq!(estimated, expected);
|
||||
}
|
||||
@@ -1465,7 +1484,70 @@ fn multiple_inline_images_apply_multiple_fixed_costs() {
|
||||
let raw_len = serde_json::to_string(&item).unwrap().len() as i64;
|
||||
let payload_sum = (payload_one.len() + payload_two.len()) as i64;
|
||||
let estimated = estimate_response_item_model_visible_bytes(&item);
|
||||
let expected = raw_len - payload_sum + (2 * IMAGE_BYTES_ESTIMATE);
|
||||
let expected = raw_len - payload_sum + (2 * RESIZED_IMAGE_BYTES_ESTIMATE);
|
||||
|
||||
assert_eq!(estimated, expected);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn original_detail_images_scale_with_dimensions() {
|
||||
// 2304x864 at 32px patches yields 72 * 27 = 1,944 patches.
|
||||
// The byte heuristic uses 4 bytes per token, so the replacement cost is 7,776 bytes.
|
||||
const EXPECTED_ORIGINAL_DETAIL_IMAGE_BYTES: i64 = 7_776;
|
||||
|
||||
let width = 2304;
|
||||
let height = 864;
|
||||
let image = ImageBuffer::from_pixel(width, height, Rgba([12u8, 34, 56, 255]));
|
||||
let mut bytes = std::io::Cursor::new(Vec::new());
|
||||
image
|
||||
.write_to(&mut bytes, ImageFormat::Png)
|
||||
.expect("encode png");
|
||||
let payload = BASE64_STANDARD.encode(bytes.get_ref());
|
||||
let image_url = format!("data:image/png;base64,{payload}");
|
||||
let item = ResponseItem::FunctionCallOutput {
|
||||
call_id: "call-original".to_string(),
|
||||
output: FunctionCallOutputPayload::from_content_items(vec![
|
||||
FunctionCallOutputContentItem::InputImage {
|
||||
image_url,
|
||||
detail: Some(ImageDetail::Original),
|
||||
},
|
||||
]),
|
||||
};
|
||||
|
||||
let raw_len = serde_json::to_string(&item).unwrap().len() as i64;
|
||||
let estimated = estimate_response_item_model_visible_bytes(&item);
|
||||
let expected = raw_len - payload.len() as i64 + EXPECTED_ORIGINAL_DETAIL_IMAGE_BYTES;
|
||||
|
||||
assert_eq!(estimated, expected);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn original_detail_webp_images_scale_with_dimensions() {
|
||||
// Same dimensions as the PNG case above, so the patch-based replacement cost is the same.
|
||||
const EXPECTED_ORIGINAL_DETAIL_IMAGE_BYTES: i64 = 7_776;
|
||||
|
||||
let width = 2304;
|
||||
let height = 864;
|
||||
let image = ImageBuffer::from_pixel(width, height, Rgba([12u8, 34, 56, 255]));
|
||||
let mut bytes = std::io::Cursor::new(Vec::new());
|
||||
image
|
||||
.write_to(&mut bytes, ImageFormat::WebP)
|
||||
.expect("encode webp");
|
||||
let payload = BASE64_STANDARD.encode(bytes.get_ref());
|
||||
let image_url = format!("data:image/webp;base64,{payload}");
|
||||
let item = ResponseItem::FunctionCallOutput {
|
||||
call_id: "call-original-webp".to_string(),
|
||||
output: FunctionCallOutputPayload::from_content_items(vec![
|
||||
FunctionCallOutputContentItem::InputImage {
|
||||
image_url,
|
||||
detail: Some(ImageDetail::Original),
|
||||
},
|
||||
]),
|
||||
};
|
||||
|
||||
let raw_len = serde_json::to_string(&item).unwrap().len() as i64;
|
||||
let estimated = estimate_response_item_model_visible_bytes(&item);
|
||||
let expected = raw_len - payload.len() as i64 + EXPECTED_ORIGINAL_DETAIL_IMAGE_BYTES;
|
||||
|
||||
assert_eq!(estimated, expected);
|
||||
}
|
||||
|
||||
@@ -119,6 +119,8 @@ pub enum Feature {
|
||||
MemoryTool,
|
||||
/// Append additional AGENTS.md guidance to user instructions.
|
||||
ChildAgentsMd,
|
||||
/// Allow `detail: "original"` image outputs on supported models.
|
||||
ImageDetailOriginal,
|
||||
/// Enforce UTF8 output in Powershell.
|
||||
PowershellUtf8,
|
||||
/// Compress request bodies (zstd) when sending streaming requests to codex-backend.
|
||||
@@ -529,6 +531,12 @@ pub const FEATURES: &[FeatureSpec] = &[
|
||||
stage: Stage::UnderDevelopment,
|
||||
default_enabled: false,
|
||||
},
|
||||
FeatureSpec {
|
||||
id: Feature::ImageDetailOriginal,
|
||||
key: "image_detail_original",
|
||||
stage: Stage::UnderDevelopment,
|
||||
default_enabled: false,
|
||||
},
|
||||
FeatureSpec {
|
||||
id: Feature::ApplyPatchFreeform,
|
||||
key: "apply_patch_freeform",
|
||||
|
||||
@@ -471,6 +471,7 @@ mod tests {
|
||||
"apply_patch_tool_type": null,
|
||||
"truncation_policy": {"mode": "bytes", "limit": 10_000},
|
||||
"supports_parallel_tool_calls": false,
|
||||
"supports_image_detail_original": false,
|
||||
"context_window": 272_000,
|
||||
"experimental_supported_tools": [],
|
||||
}))
|
||||
@@ -549,6 +550,8 @@ mod tests {
|
||||
.build()
|
||||
.await
|
||||
.expect("load default test config");
|
||||
let mut overlay = remote_model("gpt-overlay", "Overlay", 0);
|
||||
overlay.supports_image_detail_original = true;
|
||||
|
||||
let auth_manager =
|
||||
AuthManager::from_auth_for_testing(CodexAuth::from_api_key("Test API Key"));
|
||||
@@ -556,7 +559,7 @@ mod tests {
|
||||
codex_home.path().to_path_buf(),
|
||||
auth_manager,
|
||||
Some(ModelsResponse {
|
||||
models: vec![remote_model("gpt-overlay", "Overlay", 0)],
|
||||
models: vec![overlay],
|
||||
}),
|
||||
CollaborationModesConfig::default(),
|
||||
);
|
||||
@@ -568,6 +571,7 @@ mod tests {
|
||||
assert_eq!(model_info.slug, "gpt-overlay-experiment");
|
||||
assert_eq!(model_info.display_name, "Overlay");
|
||||
assert_eq!(model_info.context_window, Some(272_000));
|
||||
assert!(model_info.supports_image_detail_original);
|
||||
assert!(!model_info.supports_parallel_tool_calls);
|
||||
assert!(!model_info.used_fallback_model_metadata);
|
||||
}
|
||||
@@ -580,26 +584,24 @@ mod tests {
|
||||
.build()
|
||||
.await
|
||||
.expect("load default test config");
|
||||
let mut remote = remote_model("gpt-image", "Image", 0);
|
||||
remote.supports_image_detail_original = true;
|
||||
let auth_manager =
|
||||
AuthManager::from_auth_for_testing(CodexAuth::from_api_key("Test API Key"));
|
||||
let manager = ModelsManager::new(
|
||||
codex_home.path().to_path_buf(),
|
||||
auth_manager,
|
||||
None,
|
||||
Some(ModelsResponse {
|
||||
models: vec![remote],
|
||||
}),
|
||||
CollaborationModesConfig::default(),
|
||||
);
|
||||
let known_slug = manager
|
||||
.get_remote_models()
|
||||
.await
|
||||
.first()
|
||||
.expect("bundled models should include at least one model")
|
||||
.slug
|
||||
.clone();
|
||||
let namespaced_model = format!("custom/{known_slug}");
|
||||
let namespaced_model = "custom/gpt-image".to_string();
|
||||
|
||||
let model_info = manager.get_model_info(&namespaced_model, &config).await;
|
||||
|
||||
assert_eq!(model_info.slug, namespaced_model);
|
||||
assert!(model_info.supports_image_detail_original);
|
||||
assert!(!model_info.used_fallback_model_metadata);
|
||||
}
|
||||
|
||||
|
||||
@@ -80,6 +80,7 @@ pub(crate) fn model_info_from_slug(slug: &str) -> ModelInfo {
|
||||
apply_patch_tool_type: None,
|
||||
truncation_policy: TruncationPolicyConfig::bytes(10_000),
|
||||
supports_parallel_tool_calls: false,
|
||||
supports_image_detail_original: false,
|
||||
context_window: Some(272_000),
|
||||
auto_compact_token_limit: None,
|
||||
effective_context_window_percent: 95,
|
||||
|
||||
@@ -55,6 +55,9 @@ fn render_js_repl_instructions(config: &Config) -> Option<String> {
|
||||
section.push_str("- Helpers: `codex.tmpDir` and `codex.tool(name, args?)`.\n");
|
||||
section.push_str("- `codex.tool` executes a normal tool call and resolves to the raw tool output object. Use it for shell and non-shell tools alike.\n");
|
||||
section.push_str("- To share generated images with the model, write a file under `codex.tmpDir`, call `await codex.tool(\"view_image\", { path: \"/absolute/path\" })`, then delete the file.\n");
|
||||
if config.features.enabled(Feature::ImageDetailOriginal) {
|
||||
section.push_str("- When generating or converting images for `view_image` in `js_repl`, prefer JPEG at 85% quality unless lossless quality is strictly required; other formats can be used if the user requests them. This keeps uploads smaller and reduces the chance of hitting image size caps.\n");
|
||||
}
|
||||
section.push_str("- Top-level bindings persist across cells. If you hit `SyntaxError: Identifier 'x' has already been declared`, reuse the binding, pick a new name, wrap in `{ ... }` for block scope, or reset the kernel with `js_repl_reset`.\n");
|
||||
section.push_str("- Top-level static import declarations (for example `import x from \"pkg\"`) are currently unsupported in `js_repl`; use dynamic imports with `await import(\"pkg\")` instead.\n");
|
||||
|
||||
@@ -492,6 +495,21 @@ mod tests {
|
||||
assert_eq!(res, expected);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn js_repl_original_resolution_guidance_is_feature_gated() {
|
||||
let tmp = tempfile::tempdir().expect("tempdir");
|
||||
let mut cfg = make_config(&tmp, 4096, None).await;
|
||||
cfg.features
|
||||
.enable(Feature::JsRepl)
|
||||
.enable(Feature::ImageDetailOriginal);
|
||||
|
||||
let res = get_user_instructions(&cfg, None)
|
||||
.await
|
||||
.expect("js_repl instructions expected");
|
||||
let expected = "## JavaScript REPL (Node)\n- Use `js_repl` for Node-backed JavaScript with top-level await in a persistent kernel.\n- `js_repl` is a freeform/custom tool. Direct `js_repl` calls must send raw JavaScript tool input (optionally with first-line `// codex-js-repl: timeout_ms=15000`). Do not wrap code in JSON (for example `{\"code\":\"...\"}`), quotes, or markdown code fences.\n- Helpers: `codex.tmpDir` and `codex.tool(name, args?)`.\n- `codex.tool` executes a normal tool call and resolves to the raw tool output object. Use it for shell and non-shell tools alike.\n- To share generated images with the model, write a file under `codex.tmpDir`, call `await codex.tool(\"view_image\", { path: \"/absolute/path\" })`, then delete the file.\n- When generating or converting images for `view_image` in `js_repl`, prefer JPEG at 85% quality unless lossless quality is strictly required; other formats can be used if the user requests them. This keeps uploads smaller and reduces the chance of hitting image size caps.\n- Top-level bindings persist across cells. If you hit `SyntaxError: Identifier 'x' has already been declared`, reuse the binding, pick a new name, wrap in `{ ... }` for block scope, or reset the kernel with `js_repl_reset`.\n- Top-level static import declarations (for example `import x from \"pkg\"`) are currently unsupported in `js_repl`; use dynamic imports with `await import(\"pkg\")` instead.\n- Avoid direct access to `process.stdout` / `process.stderr` / `process.stdin`; it can corrupt the JSON line protocol. Use `console.log` and `codex.tool(...)`.";
|
||||
assert_eq!(res, expected);
|
||||
}
|
||||
|
||||
/// When both system instructions *and* a project doc are present the two
|
||||
/// should be concatenated with the separator.
|
||||
#[tokio::test]
|
||||
|
||||
@@ -222,6 +222,7 @@ mod tests {
|
||||
},
|
||||
FunctionCallOutputContentItem::InputImage {
|
||||
image_url: "data:image/png;base64,AAA".to_string(),
|
||||
detail: None,
|
||||
},
|
||||
FunctionCallOutputContentItem::InputText {
|
||||
text: "line 2".to_string(),
|
||||
@@ -239,6 +240,7 @@ mod tests {
|
||||
},
|
||||
FunctionCallOutputContentItem::InputImage {
|
||||
image_url: "data:image/png;base64,AAA".to_string(),
|
||||
detail: None,
|
||||
},
|
||||
FunctionCallOutputContentItem::InputText {
|
||||
text: "line 2".to_string(),
|
||||
|
||||
@@ -1,10 +1,15 @@
|
||||
use async_trait::async_trait;
|
||||
use codex_protocol::models::ContentItem;
|
||||
use codex_protocol::models::FunctionCallOutputBody;
|
||||
use codex_protocol::models::FunctionCallOutputContentItem;
|
||||
use codex_protocol::models::ImageDetail;
|
||||
use codex_protocol::models::local_image_content_items_with_label_number;
|
||||
use codex_protocol::openai_models::InputModality;
|
||||
use codex_utils_image::PromptImageMode;
|
||||
use serde::Deserialize;
|
||||
use tokio::fs;
|
||||
|
||||
use crate::features::Feature;
|
||||
use crate::function_tool::FunctionCallError;
|
||||
use crate::protocol::EventMsg;
|
||||
use crate::protocol::ViewImageToolCallEvent;
|
||||
@@ -14,8 +19,6 @@ use crate::tools::context::ToolPayload;
|
||||
use crate::tools::handlers::parse_arguments;
|
||||
use crate::tools::registry::ToolHandler;
|
||||
use crate::tools::registry::ToolKind;
|
||||
use codex_protocol::models::ContentItem;
|
||||
use codex_protocol::models::local_image_content_items_with_label_number;
|
||||
|
||||
pub struct ViewImageHandler;
|
||||
|
||||
@@ -81,15 +84,26 @@ impl ToolHandler for ViewImageHandler {
|
||||
}
|
||||
let event_path = abs_path.clone();
|
||||
|
||||
let content = local_image_content_items_with_label_number(&abs_path, None);
|
||||
let content = content
|
||||
let use_original_detail = turn.config.features.enabled(Feature::ImageDetailOriginal)
|
||||
&& turn.model_info.supports_image_detail_original;
|
||||
let image_mode = if use_original_detail {
|
||||
PromptImageMode::Original
|
||||
} else {
|
||||
PromptImageMode::ResizeToFit
|
||||
};
|
||||
let image_detail = use_original_detail.then_some(ImageDetail::Original);
|
||||
|
||||
let content = local_image_content_items_with_label_number(&abs_path, None, image_mode)
|
||||
.into_iter()
|
||||
.map(|item| match item {
|
||||
ContentItem::InputText { text } => {
|
||||
FunctionCallOutputContentItem::InputText { text }
|
||||
}
|
||||
ContentItem::InputImage { image_url } => {
|
||||
FunctionCallOutputContentItem::InputImage { image_url }
|
||||
FunctionCallOutputContentItem::InputImage {
|
||||
image_url,
|
||||
detail: image_detail,
|
||||
}
|
||||
}
|
||||
ContentItem::OutputText { text } => {
|
||||
FunctionCallOutputContentItem::InputText { text }
|
||||
|
||||
@@ -1900,6 +1900,7 @@ mod tests {
|
||||
output: FunctionCallOutputPayload::from_content_items(vec![
|
||||
FunctionCallOutputContentItem::InputImage {
|
||||
image_url: "data:image/png;base64,abcd".to_string(),
|
||||
detail: None,
|
||||
},
|
||||
]),
|
||||
};
|
||||
@@ -1929,6 +1930,7 @@ mod tests {
|
||||
output: FunctionCallOutputPayload::from_content_items(vec![
|
||||
FunctionCallOutputContentItem::InputImage {
|
||||
image_url: "data:image/png;base64,abcd".to_string(),
|
||||
detail: None,
|
||||
},
|
||||
]),
|
||||
};
|
||||
@@ -2417,15 +2419,17 @@ console.log(out.output?.body?.text ?? "");
|
||||
image_url:
|
||||
"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/iZk9HQAAAABJRU5ErkJggg=="
|
||||
.to_string(),
|
||||
detail: None,
|
||||
}]
|
||||
.as_slice()
|
||||
);
|
||||
let [FunctionCallOutputContentItem::InputImage { image_url }] =
|
||||
let [FunctionCallOutputContentItem::InputImage { image_url, detail }] =
|
||||
result.content_items.as_slice()
|
||||
else {
|
||||
panic!("view_image should return exactly one input_image content item");
|
||||
};
|
||||
assert!(image_url.starts_with("data:image/png;base64,"));
|
||||
assert_eq!(*detail, None);
|
||||
assert!(session.get_pending_input().await.is_empty());
|
||||
|
||||
Ok(())
|
||||
@@ -2515,6 +2519,7 @@ console.log(out.type);
|
||||
},
|
||||
FunctionCallOutputContentItem::InputImage {
|
||||
image_url: image_url.to_string(),
|
||||
detail: None,
|
||||
},
|
||||
]
|
||||
);
|
||||
|
||||
@@ -138,9 +138,10 @@ pub(crate) fn truncate_function_output_items_with_policy(
|
||||
remaining_budget = 0;
|
||||
}
|
||||
}
|
||||
FunctionCallOutputContentItem::InputImage { image_url } => {
|
||||
FunctionCallOutputContentItem::InputImage { image_url, detail } => {
|
||||
out.push(FunctionCallOutputContentItem::InputImage {
|
||||
image_url: image_url.clone(),
|
||||
detail: *detail,
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -491,6 +492,7 @@ mod tests {
|
||||
FunctionCallOutputContentItem::InputText { text: t2.clone() },
|
||||
FunctionCallOutputContentItem::InputImage {
|
||||
image_url: "img:mid".to_string(),
|
||||
detail: None,
|
||||
},
|
||||
FunctionCallOutputContentItem::InputText { text: t3 },
|
||||
FunctionCallOutputContentItem::InputText { text: t4 },
|
||||
@@ -518,7 +520,8 @@ mod tests {
|
||||
assert_eq!(
|
||||
output[2],
|
||||
FunctionCallOutputContentItem::InputImage {
|
||||
image_url: "img:mid".to_string()
|
||||
image_url: "img:mid".to_string(),
|
||||
detail: None,
|
||||
}
|
||||
);
|
||||
|
||||
|
||||
Reference in New Issue
Block a user