[Codex][CLI] Gate image inputs by model modalities (#10271)

###### Summary

- Add input_modalities to model metadata so clients can determine
supported input types.
- Gate image paste/attach in TUI when the selected model does not
support images.
- Block submits that include images for unsupported models and show a
clear warning.
- Propagate modality metadata through app-server protocol/model-list
responses.
  - Update related tests/fixtures.

  ###### Rationale

  - Models support different input modalities.
- Clients need an explicit capability signal to prevent unsupported
requests.
- Backward-compatible defaults preserve existing behavior when modality
metadata is absent.

  ###### Scope

  - codex-rs/protocol, codex-rs/core, codex-rs/tui
  - codex-rs/app-server-protocol, codex-rs/app-server
  - Generated app-server types / schema fixtures

  ###### Trade-offs

- Default behavior assumes text + image when field is absent for
compatibility.
  - Server-side validation remains the source of truth.

  ###### Follow-up

- Non-TUI clients should consume input_modalities to disable unsupported
attachments.
- Model catalogs should explicitly set input_modalities for text-only
models.

  ###### Testing

  - cargo fmt --all
  - cargo test -p codex-tui
  - env -u GITHUB_APP_KEY cargo test -p codex-core --lib
  - just write-app-server-schema
- cargo run -p codex-cli --bin codex -- app-server generate-ts --out
app-server-types
  - test against local backend
  
<img width="695" height="199" alt="image"
src="https://github.com/user-attachments/assets/d22dd04f-5eba-4db9-a7c5-a2506f60ec44"
/>

---------

Co-authored-by: Josh McKinney <joshka@openai.com>
This commit is contained in:
Colin Young
2026-02-02 18:56:39 -08:00
committed by GitHub
parent b8addcddb9
commit 7e07ec8f73
23 changed files with 373 additions and 3 deletions

View File

@@ -70,6 +70,7 @@ use codex_protocol::config_types::Personality;
use codex_protocol::config_types::Settings;
use codex_protocol::openai_models::ModelPreset;
use codex_protocol::openai_models::ReasoningEffortPreset;
use codex_protocol::openai_models::default_input_modalities;
use codex_protocol::parse_command::ParsedCommand;
use codex_protocol::plan_tool::PlanItemArg;
use codex_protocol::plan_tool::StepStatus;
@@ -324,6 +325,49 @@ async fn submission_preserves_text_elements_and_local_images() {
assert_eq!(stored_images, local_images);
}
#[tokio::test]
async fn blocked_image_restore_preserves_mention_paths() {
let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(None).await;
let placeholder = "[Image #1]";
let text = format!("{placeholder} check $file");
let text_elements = vec![TextElement::new(
(0..placeholder.len()).into(),
Some(placeholder.to_string()),
)];
let local_images = vec![LocalImageAttachment {
placeholder: placeholder.to_string(),
path: PathBuf::from("/tmp/blocked.png"),
}];
let mention_paths =
HashMap::from([("file".to_string(), "/tmp/skills/file/SKILL.md".to_string())]);
chat.restore_blocked_image_submission(
text.clone(),
text_elements.clone(),
local_images.clone(),
mention_paths.clone(),
);
assert_eq!(chat.bottom_pane.composer_text(), text);
assert_eq!(chat.bottom_pane.composer_text_elements(), text_elements);
assert_eq!(
chat.bottom_pane.composer_local_image_paths(),
vec![local_images[0].path.clone()],
);
assert_eq!(chat.bottom_pane.take_mention_paths(), mention_paths);
let cells = drain_insert_history(&mut rx);
let warning = cells
.last()
.map(|lines| lines_to_single_string(lines))
.expect("expected warning cell");
assert!(
warning.contains("does not support image inputs"),
"expected image warning, got: {warning:?}"
);
}
#[tokio::test]
async fn interrupted_turn_restores_queued_messages_with_images_and_elements() {
let (mut chat, _rx, _op_rx) = make_chatwidget_manual(None).await;
@@ -3154,6 +3198,7 @@ async fn model_picker_hides_show_in_picker_false_models_from_cache() {
upgrade: None,
show_in_picker,
supported_in_api: true,
input_modalities: default_input_modalities(),
};
chat.open_model_popup_with_presets(vec![
@@ -3392,6 +3437,7 @@ async fn single_reasoning_option_skips_selection() {
upgrade: None,
show_in_picker: true,
supported_in_api: true,
input_modalities: default_input_modalities(),
};
chat.open_reasoning_popup(preset);