Compare commits

...

3 Commits

Author SHA1 Message Date
Sayan Sisodiya
b4f639b81b use bundled model info when remote_models disabled 2026-02-12 22:18:36 -08:00
Sayan Sisodiya
40b44a5361 brittle test fix 2026-02-12 21:48:16 -08:00
Sayan Sisodiya
07e0fe9986 show user warning when using default fallback metadata, as it can break things 2026-02-12 21:38:23 -08:00
4 changed files with 155 additions and 17 deletions

View File

@@ -1628,6 +1628,34 @@ impl Session {
state.set_previous_model(previous_model);
}
async fn should_emit_unknown_model_warning(&self, model_slug: &str) -> bool {
let mut state = self.state.lock().await;
state.mark_unknown_model_warning_emitted(model_slug)
}
async fn maybe_warn_for_unknown_model(
&self,
turn_context: &TurnContext,
model_slug: &str,
used_fallback_model_metadata: bool,
) {
if !used_fallback_model_metadata
|| !self.should_emit_unknown_model_warning(model_slug).await
{
return;
}
self.send_event(
turn_context,
EventMsg::Warning(WarningEvent {
message: format!(
"Defaulting to fallback model metadata for `{model_slug}`. This can degrade performance and cause issues."
),
}),
)
.await;
}
fn maybe_refresh_shell_snapshot_for_cwd(
&self,
previous_cwd: &Path,
@@ -1754,13 +1782,11 @@ impl Session {
}
}
let model_info = self
let resolved_model_slug = session_configuration.collaboration_mode.model().to_string();
let model_resolution = self
.services
.models_manager
.get_model_info(
session_configuration.collaboration_mode.model(),
&per_turn_config,
)
.get_model_info_resolution(resolved_model_slug.as_str(), &per_turn_config)
.await;
let mut turn_context: TurnContext = Self::make_turn_context(
Some(Arc::clone(&self.services.auth_manager)),
@@ -1768,7 +1794,7 @@ impl Session {
session_configuration.provider.clone(),
&session_configuration,
per_turn_config,
model_info,
model_resolution.model_info,
self.services
.network_proxy
.as_ref()
@@ -1781,6 +1807,12 @@ impl Session {
turn_context.final_output_json_schema = final_schema;
}
let turn_context = Arc::new(turn_context);
self.maybe_warn_for_unknown_model(
turn_context.as_ref(),
resolved_model_slug.as_str(),
model_resolution.used_fallback_model_metadata,
)
.await;
turn_context.spawn_turn_metadata_header_task();
turn_context
}

View File

@@ -43,10 +43,18 @@ pub enum RefreshStrategy {
OnlineIfUncached,
}
/// Result of resolving model metadata for a requested model slug.
#[derive(Debug, Clone)]
pub(crate) struct ModelInfoResolution {
pub(crate) model_info: ModelInfo,
pub(crate) used_fallback_model_metadata: bool,
}
/// Coordinates remote model discovery plus cached metadata on disk.
#[derive(Debug)]
pub struct ModelsManager {
local_models: Vec<ModelPreset>,
bundled_models: Vec<ModelInfo>,
remote_models: RwLock<Vec<ModelInfo>>,
auth_manager: Arc<AuthManager>,
etag: RwLock<Option<String>>,
@@ -61,9 +69,11 @@ impl ModelsManager {
pub fn new(codex_home: PathBuf, auth_manager: Arc<AuthManager>) -> Self {
let cache_path = codex_home.join(MODEL_CACHE_FILE);
let cache_manager = ModelsCacheManager::new(cache_path, DEFAULT_MODEL_CACHE_TTL);
let bundled_models = Self::load_remote_models_from_file().unwrap_or_default();
Self {
local_models: builtin_model_presets(auth_manager.auth_mode()),
remote_models: RwLock::new(Self::load_remote_models_from_file().unwrap_or_default()),
bundled_models: bundled_models.clone(),
remote_models: RwLock::new(bundled_models),
auth_manager,
etag: RwLock::new(None),
cache_manager,
@@ -137,18 +147,35 @@ impl ModelsManager {
// todo(aibrahim): look if we can tighten it to pub(crate)
/// Look up model metadata, applying remote overrides and config adjustments.
pub async fn get_model_info(&self, model: &str, config: &Config) -> ModelInfo {
self.get_model_info_resolution(model, config)
.await
.model_info
}
/// Look up model metadata, returning whether fallback metadata was used.
pub(crate) async fn get_model_info_resolution(
&self,
model: &str,
config: &Config,
) -> ModelInfoResolution {
let remote = self
.find_remote_model_by_longest_prefix(model, config)
.await;
let model = if let Some(remote) = remote {
ModelInfo {
slug: model.to_string(),
..remote
}
let (model_info, used_fallback_model_metadata) = if let Some(remote) = remote {
(
ModelInfo {
slug: model.to_string(),
..remote
},
false,
)
} else {
model_info::model_info_from_slug(model)
(model_info::model_info_from_slug(model), true)
};
model_info::with_config_overrides(model, config)
ModelInfoResolution {
model_info: model_info::with_config_overrides(model_info, config),
used_fallback_model_metadata,
}
}
async fn find_remote_model_by_longest_prefix(
@@ -156,8 +183,13 @@ impl ModelsManager {
model: &str,
config: &Config,
) -> Option<ModelInfo> {
let models_for_resolution = if config.features.enabled(Feature::RemoteModels) {
self.remote_models.read().await.clone()
} else {
self.bundled_models.clone()
};
let mut best: Option<ModelInfo> = None;
for candidate in self.get_remote_models(config).await {
for candidate in models_for_resolution {
if !model.starts_with(&candidate.slug) {
continue;
}
@@ -359,9 +391,11 @@ impl ModelsManager {
) -> Self {
let cache_path = codex_home.join(MODEL_CACHE_FILE);
let cache_manager = ModelsCacheManager::new(cache_path, DEFAULT_MODEL_CACHE_TTL);
let bundled_models = Self::load_remote_models_from_file().unwrap_or_default();
Self {
local_models: builtin_model_presets(auth_manager.auth_mode()),
remote_models: RwLock::new(Self::load_remote_models_from_file().unwrap_or_default()),
bundled_models: bundled_models.clone(),
remote_models: RwLock::new(bundled_models),
auth_manager,
etag: RwLock::new(None),
cache_manager,
@@ -472,6 +506,52 @@ mod tests {
}
}
#[tokio::test]
async fn get_model_info_resolution_tracks_fallback_usage() {
let codex_home = tempdir().expect("temp dir");
let mut config = ConfigBuilder::default()
.codex_home(codex_home.path().to_path_buf())
.build()
.await
.expect("load default test config");
config.features.enable(Feature::RemoteModels);
let mut config_with_remote_models_disabled = config.clone();
config_with_remote_models_disabled
.features
.disable(Feature::RemoteModels);
let auth_manager =
AuthManager::from_auth_for_testing(CodexAuth::from_api_key("Test API Key"));
let manager = ModelsManager::new(codex_home.path().to_path_buf(), auth_manager);
let known_slug = manager
.get_remote_models(&config)
.await
.first()
.expect("bundled models should include at least one model")
.slug
.clone();
let known = manager
.get_model_info_resolution(known_slug.as_str(), &config)
.await;
assert!(!known.used_fallback_model_metadata);
assert_eq!(known.model_info.slug, known_slug.as_str());
let known_without_remote_models = manager
.get_model_info_resolution(known_slug.as_str(), &config_with_remote_models_disabled)
.await;
assert!(!known_without_remote_models.used_fallback_model_metadata);
assert_eq!(
known_without_remote_models.model_info.slug,
known_slug.as_str()
);
let unknown = manager
.get_model_info_resolution("model-that-does-not-exist", &config)
.await;
assert!(unknown.used_fallback_model_metadata);
assert_eq!(unknown.model_info.slug, "model-that-does-not-exist");
}
#[tokio::test]
async fn refresh_available_models_sorts_by_priority() {
let server = MockServer::start().await;

View File

@@ -31,6 +31,7 @@ pub(crate) struct SessionState {
pub(crate) startup_regular_task: Option<RegularTask>,
pub(crate) active_mcp_tool_selection: Option<Vec<String>>,
pub(crate) active_connector_selection: HashSet<String>,
warned_unknown_models: HashSet<String>,
}
impl SessionState {
@@ -49,6 +50,7 @@ impl SessionState {
startup_regular_task: None,
active_mcp_tool_selection: None,
active_connector_selection: HashSet::new(),
warned_unknown_models: HashSet::new(),
}
}
@@ -196,6 +198,13 @@ impl SessionState {
pub(crate) fn clear_connector_selection(&mut self) {
self.active_connector_selection.clear();
}
/// Marks an unknown model warning as emitted for the provided slug.
///
/// Returns `true` only the first time a slug is marked during this session.
pub(crate) fn mark_unknown_model_warning_emitted(&mut self, model_slug: &str) -> bool {
self.warned_unknown_models.insert(model_slug.to_string())
}
}
// Sometimes new snapshots don't include credits or plan information.
@@ -322,6 +331,16 @@ mod tests {
assert_eq!(state.get_connector_selection(), HashSet::new());
}
#[tokio::test]
async fn unknown_model_warning_tracking_deduplicates_by_slug() {
let session_configuration = make_session_configuration_for_tests().await;
let mut state = SessionState::new(session_configuration);
assert!(state.mark_unknown_model_warning_emitted("gpt-alpha"));
assert!(!state.mark_unknown_model_warning_emitted("gpt-alpha"));
assert!(state.mark_unknown_model_warning_emitted("gpt-beta"));
}
#[tokio::test]
async fn set_rate_limits_defaults_limit_id_to_codex_when_missing() {
let session_configuration = make_session_configuration_for_tests().await;

View File

@@ -74,7 +74,14 @@ async fn emits_warning_when_resumed_model_differs() {
.expect("resume conversation");
// Assert: a Warning event is emitted describing the model mismatch.
let warning = wait_for_event(&conversation, |ev| matches!(ev, EventMsg::Warning(_))).await;
let warning = wait_for_event(&conversation, |ev| {
matches!(
ev,
EventMsg::Warning(WarningEvent { message })
if message.contains("previous-model") && message.contains("current-model")
)
})
.await;
let EventMsg::Warning(WarningEvent { message }) = warning else {
panic!("expected warning event");
};