fix: show user warning when using default fallback metadata (#11690)

### What
It's currently unclear when the harness falls back to the default,
generic `ModelInfo`. This happens when the `remote_models` feature is
disabled or the model is truly unknown, and can lead to bad performance
and issues in the harness.

Add a user-facing warning when this happens so they are aware when their
setup is broken.

### Tests
Added tests, tested locally.
This commit is contained in:
sayan-oai
2026-02-15 18:46:05 -08:00
committed by GitHub
parent 85034b189e
commit 060a320e7d
13 changed files with 76 additions and 8 deletions

View File

@@ -1780,13 +1780,11 @@ impl Session {
}
}
let resolved_model_slug = session_configuration.collaboration_mode.model().to_string();
let model_info = self
.services
.models_manager
.get_model_info(
session_configuration.collaboration_mode.model(),
&per_turn_config,
)
.get_model_info(resolved_model_slug.as_str(), &per_turn_config)
.await;
let mut turn_context: TurnContext = Self::make_turn_context(
Some(Arc::clone(&self.services.auth_manager)),
@@ -1794,7 +1792,7 @@ impl Session {
session_configuration.provider.clone(),
&session_configuration,
per_turn_config,
model_info,
model_info.clone(),
self.services
.network_proxy
.as_ref()
@@ -1807,6 +1805,17 @@ impl Session {
turn_context.final_output_json_schema = final_schema;
}
let turn_context = Arc::new(turn_context);
if model_info.used_fallback_model_metadata {
self.send_event(
turn_context.as_ref(),
EventMsg::Warning(WarningEvent {
message: format!(
"Model metadata for `{resolved_model_slug}` not found. Defaulting to fallback metadata; this can degrade performance and cause issues."
),
}),
)
.await;
}
turn_context.turn_metadata_state.spawn_git_enrichment_task();
turn_context
}

View File

@@ -140,15 +140,16 @@ impl ModelsManager {
let remote = self
.find_remote_model_by_longest_prefix(model, config)
.await;
let model = if let Some(remote) = remote {
let model_info = if let Some(remote) = remote {
ModelInfo {
slug: model.to_string(),
used_fallback_model_metadata: false,
..remote
}
} else {
model_info::model_info_from_slug(model)
};
model_info::with_config_overrides(model, config)
model_info::with_config_overrides(model_info, config)
}
async fn find_remote_model_by_longest_prefix(
@@ -472,6 +473,37 @@ mod tests {
}
}
#[tokio::test]
async fn get_model_info_tracks_fallback_usage() {
let codex_home = tempdir().expect("temp dir");
let mut config = ConfigBuilder::default()
.codex_home(codex_home.path().to_path_buf())
.build()
.await
.expect("load default test config");
config.features.enable(Feature::RemoteModels);
let auth_manager =
AuthManager::from_auth_for_testing(CodexAuth::from_api_key("Test API Key"));
let manager = ModelsManager::new(codex_home.path().to_path_buf(), auth_manager);
let known_slug = manager
.get_remote_models(&config)
.await
.first()
.expect("bundled models should include at least one model")
.slug
.clone();
let known = manager.get_model_info(known_slug.as_str(), &config).await;
assert!(!known.used_fallback_model_metadata);
assert_eq!(known.slug, known_slug);
let unknown = manager
.get_model_info("model-that-does-not-exist", &config)
.await;
assert!(unknown.used_fallback_model_metadata);
assert_eq!(unknown.slug, "model-that-does-not-exist");
}
#[tokio::test]
async fn refresh_available_models_sorts_by_priority() {
let server = MockServer::start().await;

View File

@@ -81,6 +81,7 @@ pub(crate) fn model_info_from_slug(slug: &str) -> ModelInfo {
experimental_supported_tools: Vec::new(),
input_modalities: default_input_modalities(),
prefer_websockets: false,
used_fallback_model_metadata: true, // this is the fallback model metadata
}
}