mirror of
https://github.com/openai/codex.git
synced 2026-05-05 11:57:33 +00:00
[Codex][CLI] Gate image inputs by model modalities (#10271)
###### Summary - Add input_modalities to model metadata so clients can determine supported input types. - Gate image paste/attach in TUI when the selected model does not support images. - Block submits that include images for unsupported models and show a clear warning. - Propagate modality metadata through app-server protocol/model-list responses. - Update related tests/fixtures. ###### Rationale - Models support different input modalities. - Clients need an explicit capability signal to prevent unsupported requests. - Backward-compatible defaults preserve existing behavior when modality metadata is absent. ###### Scope - codex-rs/protocol, codex-rs/core, codex-rs/tui - codex-rs/app-server-protocol, codex-rs/app-server - Generated app-server types / schema fixtures ###### Trade-offs - Default behavior assumes text + image when field is absent for compatibility. - Server-side validation remains the source of truth. ###### Follow-up - Non-TUI clients should consume input_modalities to disable unsupported attachments. - Model catalogs should explicitly set input_modalities for text-only models. ###### Testing - cargo fmt --all - cargo test -p codex-tui - env -u GITHUB_APP_KEY cargo test -p codex-core --lib - just write-app-server-schema - cargo run -p codex-cli --bin codex -- app-server generate-ts --out app-server-types - test against local backend <img width="695" height="199" alt="image" src="https://github.com/user-attachments/assets/d22dd04f-5eba-4db9-a7c5-a2506f60ec44" /> --------- Co-authored-by: Josh McKinney <joshka@openai.com>
This commit is contained in:
@@ -70,6 +70,7 @@ use codex_protocol::config_types::Personality;
|
||||
use codex_protocol::config_types::Settings;
|
||||
use codex_protocol::openai_models::ModelPreset;
|
||||
use codex_protocol::openai_models::ReasoningEffortPreset;
|
||||
use codex_protocol::openai_models::default_input_modalities;
|
||||
use codex_protocol::parse_command::ParsedCommand;
|
||||
use codex_protocol::plan_tool::PlanItemArg;
|
||||
use codex_protocol::plan_tool::StepStatus;
|
||||
@@ -324,6 +325,49 @@ async fn submission_preserves_text_elements_and_local_images() {
|
||||
assert_eq!(stored_images, local_images);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn blocked_image_restore_preserves_mention_paths() {
|
||||
let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(None).await;
|
||||
|
||||
let placeholder = "[Image #1]";
|
||||
let text = format!("{placeholder} check $file");
|
||||
let text_elements = vec![TextElement::new(
|
||||
(0..placeholder.len()).into(),
|
||||
Some(placeholder.to_string()),
|
||||
)];
|
||||
let local_images = vec![LocalImageAttachment {
|
||||
placeholder: placeholder.to_string(),
|
||||
path: PathBuf::from("/tmp/blocked.png"),
|
||||
}];
|
||||
let mention_paths =
|
||||
HashMap::from([("file".to_string(), "/tmp/skills/file/SKILL.md".to_string())]);
|
||||
|
||||
chat.restore_blocked_image_submission(
|
||||
text.clone(),
|
||||
text_elements.clone(),
|
||||
local_images.clone(),
|
||||
mention_paths.clone(),
|
||||
);
|
||||
|
||||
assert_eq!(chat.bottom_pane.composer_text(), text);
|
||||
assert_eq!(chat.bottom_pane.composer_text_elements(), text_elements);
|
||||
assert_eq!(
|
||||
chat.bottom_pane.composer_local_image_paths(),
|
||||
vec![local_images[0].path.clone()],
|
||||
);
|
||||
assert_eq!(chat.bottom_pane.take_mention_paths(), mention_paths);
|
||||
|
||||
let cells = drain_insert_history(&mut rx);
|
||||
let warning = cells
|
||||
.last()
|
||||
.map(|lines| lines_to_single_string(lines))
|
||||
.expect("expected warning cell");
|
||||
assert!(
|
||||
warning.contains("does not support image inputs"),
|
||||
"expected image warning, got: {warning:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn interrupted_turn_restores_queued_messages_with_images_and_elements() {
|
||||
let (mut chat, _rx, _op_rx) = make_chatwidget_manual(None).await;
|
||||
@@ -3154,6 +3198,7 @@ async fn model_picker_hides_show_in_picker_false_models_from_cache() {
|
||||
upgrade: None,
|
||||
show_in_picker,
|
||||
supported_in_api: true,
|
||||
input_modalities: default_input_modalities(),
|
||||
};
|
||||
|
||||
chat.open_model_popup_with_presets(vec![
|
||||
@@ -3392,6 +3437,7 @@ async fn single_reasoning_option_skips_selection() {
|
||||
upgrade: None,
|
||||
show_in_picker: true,
|
||||
supported_in_api: true,
|
||||
input_modalities: default_input_modalities(),
|
||||
};
|
||||
chat.open_reasoning_popup(preset);
|
||||
|
||||
|
||||
Reference in New Issue
Block a user