Preserve image detail in app-server inputs (#20693)

## Summary

- Add optional image detail to user image inputs across core, app-server
v2, thread history/event mapping, and the generated app-server
schemas/types.
- Preserve requested detail when serializing Responses image inputs:
omitted detail stays on the existing `high` default, while explicit
`original` keeps local images on the original-resolution path.
- Support `high`/`original` consistently for tool image outputs,
including MCP `codex/imageDetail`, code-mode image helpers, and
`view_image`.
This commit is contained in:
Curtis 'Fjord' Hawthorne
2026-05-15 15:04:04 -07:00
committed by GitHub
parent 249d50aafc
commit 8543e39885
81 changed files with 1302 additions and 156 deletions

View File

@@ -15,6 +15,7 @@ use codex_protocol::memory_citation::MemoryCitation as CoreMemoryCitation;
use codex_protocol::memory_citation::MemoryCitationEntry as CoreMemoryCitationEntry;
use codex_protocol::models::AdditionalPermissionProfile as CoreAdditionalPermissionProfile;
use codex_protocol::models::FileSystemPermissions as CoreFileSystemPermissions;
use codex_protocol::models::ImageDetail;
use codex_protocol::models::ManagedFileSystemPermissions as CoreManagedFileSystemPermissions;
use codex_protocol::models::MessagePhase;
use codex_protocol::models::NetworkPermissions as CoreNetworkPermissions;
@@ -2319,9 +2320,11 @@ fn core_turn_item_into_thread_item_converts_supported_variants() {
},
CoreUserInput::Image {
image_url: "https://example.com/image.png".to_string(),
detail: Some(ImageDetail::Original),
},
CoreUserInput::LocalImage {
path: PathBuf::from("local/image.png"),
detail: Some(ImageDetail::Original),
},
CoreUserInput::Skill {
name: "skill-creator".to_string(),
@@ -2345,9 +2348,11 @@ fn core_turn_item_into_thread_item_converts_supported_variants() {
},
UserInput::Image {
url: "https://example.com/image.png".to_string(),
detail: Some(ImageDetail::Original),
},
UserInput::LocalImage {
path: PathBuf::from("local/image.png"),
detail: Some(ImageDetail::Original),
},
UserInput::Skill {
name: "skill-creator".to_string(),
@@ -2562,6 +2567,33 @@ fn core_turn_item_into_thread_item_converts_supported_variants() {
);
}
#[test]
fn user_input_into_core_preserves_image_detail() {
assert_eq!(
UserInput::Image {
url: "https://example.com/image.png".to_string(),
detail: Some(ImageDetail::Original),
}
.into_core(),
CoreUserInput::Image {
image_url: "https://example.com/image.png".to_string(),
detail: Some(ImageDetail::Original),
}
);
assert_eq!(
UserInput::LocalImage {
path: PathBuf::from("local/image.png"),
detail: Some(ImageDetail::Original),
}
.into_core(),
CoreUserInput::LocalImage {
path: PathBuf::from("local/image.png"),
detail: Some(ImageDetail::Original),
}
);
}
#[test]
fn skills_list_params_serialization_uses_force_reload() {
assert_eq!(

View File

@@ -7,6 +7,7 @@ use codex_experimental_api_macros::ExperimentalApi;
use codex_protocol::config_types::CollaborationMode;
use codex_protocol::config_types::Personality;
use codex_protocol::config_types::ReasoningSummary;
use codex_protocol::models::ImageDetail;
use codex_protocol::openai_models::ReasoningEffort;
use codex_protocol::plan_tool::PlanItemArg as CorePlanItemArg;
use codex_protocol::plan_tool::StepStatus as CorePlanStepStatus;
@@ -249,9 +250,15 @@ pub enum UserInput {
text_elements: Vec<TextElement>,
},
Image {
#[serde(default)]
#[ts(optional)]
detail: Option<ImageDetail>,
url: String,
},
LocalImage {
#[serde(default)]
#[ts(optional)]
detail: Option<ImageDetail>,
path: PathBuf,
},
Skill {
@@ -274,8 +281,11 @@ impl UserInput {
text,
text_elements: text_elements.into_iter().map(Into::into).collect(),
},
UserInput::Image { url } => CoreUserInput::Image { image_url: url },
UserInput::LocalImage { path } => CoreUserInput::LocalImage { path },
UserInput::Image { url, detail } => CoreUserInput::Image {
image_url: url,
detail,
},
UserInput::LocalImage { path, detail } => CoreUserInput::LocalImage { path, detail },
UserInput::Skill { name, path } => CoreUserInput::Skill { name, path },
UserInput::Mention { name, path } => CoreUserInput::Mention { name, path },
}
@@ -292,8 +302,11 @@ impl From<CoreUserInput> for UserInput {
text,
text_elements: text_elements.into_iter().map(Into::into).collect(),
},
CoreUserInput::Image { image_url } => UserInput::Image { url: image_url },
CoreUserInput::LocalImage { path } => UserInput::LocalImage { path },
CoreUserInput::Image { image_url, detail } => UserInput::Image {
url: image_url,
detail,
},
CoreUserInput::LocalImage { path, detail } => UserInput::LocalImage { path, detail },
CoreUserInput::Skill { name, path } => UserInput::Skill { name, path },
CoreUserInput::Mention { name, path } => UserInput::Mention { name, path },
_ => unreachable!("unsupported user input variant"),