From 028e32d35e51073efcc5cf328b0fb328c9e6f626 Mon Sep 17 00:00:00 2001 From: won Date: Wed, 20 May 2026 16:40:14 -0700 Subject: [PATCH] move integration test to a new .rs file --- codex-rs/core/tests/suite/auto_review.rs | 240 +++++++++++++++++++++ codex-rs/core/tests/suite/mod.rs | 1 + codex-rs/core/tests/suite/remote_models.rs | 170 --------------- 3 files changed, 241 insertions(+), 170 deletions(-) create mode 100644 codex-rs/core/tests/suite/auto_review.rs diff --git a/codex-rs/core/tests/suite/auto_review.rs b/codex-rs/core/tests/suite/auto_review.rs new file mode 100644 index 0000000000..b285a2ee03 --- /dev/null +++ b/codex-rs/core/tests/suite/auto_review.rs @@ -0,0 +1,240 @@ +#![allow(clippy::expect_used)] + +use anyhow::Result; +use codex_features::Feature; +use codex_login::CodexAuth; +use codex_models_manager::manager::RefreshStrategy; +use codex_protocol::config_types::ApprovalsReviewer; +use codex_protocol::config_types::ReasoningSummary; +use codex_protocol::models::PermissionProfile; +use codex_protocol::openai_models::ConfigShellToolType; +use codex_protocol::openai_models::ModelInfo; +use codex_protocol::openai_models::ModelVisibility; +use codex_protocol::openai_models::ModelsResponse; +use codex_protocol::openai_models::ReasoningEffort; +use codex_protocol::openai_models::ReasoningEffortPreset; +use codex_protocol::openai_models::TruncationPolicyConfig; +use codex_protocol::openai_models::default_input_modalities; +use codex_protocol::protocol::AskForApproval; +use codex_protocol::protocol::EventMsg; +use codex_protocol::protocol::Op; +use codex_protocol::request_permissions::PermissionGrantScope; +use codex_protocol::request_permissions::RequestPermissionsResponse; +use codex_protocol::user_input::UserInput; +use core_test_support::responses::ev_apply_patch_custom_tool_call; +use core_test_support::responses::ev_assistant_message; +use core_test_support::responses::ev_completed; +use core_test_support::responses::ev_function_call; +use core_test_support::responses::ev_response_created; +use core_test_support::responses::mount_models_once; +use core_test_support::responses::mount_sse_sequence; +use core_test_support::responses::sse; +use core_test_support::skip_if_no_network; +use core_test_support::skip_if_sandbox; +use core_test_support::test_codex::TestCodex; +use core_test_support::test_codex::test_codex; +use core_test_support::test_codex::turn_permission_fields; +use core_test_support::wait_for_event; +use pretty_assertions::assert_eq; +use serde_json::json; +use wiremock::MockServer; + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn remote_model_override_uses_parent_model_for_strict_auto_review() -> Result<()> { + skip_if_no_network!(Ok(())); + skip_if_sandbox!(Ok(())); + + let server = MockServer::start().await; + let model = "remote-auto-review-parent"; + mount_models_once( + &server, + ModelsResponse { + models: vec![remote_model_with_auto_review_override(model)], + }, + ) + .await; + + let permissions_call_id = "auto-review-permissions-call"; + let permissions_args = json!({ + "reason": "exercise strict Guardian model selection", + "permissions": { + "network": { + "enabled": true, + }, + }, + }); + let patch_call_id = "auto-review-patch-call"; + let patch = "*** Begin Patch\n*** Add File: auto-review-model-override.txt\n+exercise Guardian model selection\n*** End Patch\n"; + let responses = mount_sse_sequence( + &server, + vec![ + sse(vec![ + ev_response_created("resp-parent-1"), + ev_function_call( + permissions_call_id, + "request_permissions", + &serde_json::to_string(&permissions_args)?, + ), + ev_completed("resp-parent-1"), + ]), + sse(vec![ + ev_response_created("resp-parent-2"), + ev_apply_patch_custom_tool_call(patch_call_id, patch), + ev_completed("resp-parent-2"), + ]), + sse(vec![ + ev_response_created("resp-guardian"), + ev_assistant_message( + "msg-guardian", + &json!({ + "risk_level": "low", + "user_authorization": "high", + "outcome": "allow", + "rationale": "The patch only exercises Guardian model selection.", + }) + .to_string(), + ), + ev_completed("resp-guardian"), + ]), + sse(vec![ + ev_response_created("resp-parent-3"), + ev_assistant_message("msg-parent-3", "done"), + ev_completed("resp-parent-3"), + ]), + ], + ) + .await; + + let mut builder = test_codex() + .with_auth(CodexAuth::create_dummy_chatgpt_auth_for_testing()) + .with_config(|config| { + config.model = Some("gpt-5.4".to_string()); + config.approvals_reviewer = ApprovalsReviewer::User; + config + .features + .enable(Feature::ExecPermissionApprovals) + .expect("test config should allow feature update"); + config + .features + .enable(Feature::RequestPermissionsTool) + .expect("test config should allow feature update"); + }); + let TestCodex { + codex, + cwd, + config, + thread_manager, + .. + } = builder.build(&server).await?; + + let models_manager = thread_manager.get_models_manager(); + models_manager + .list_models(RefreshStrategy::OnlineIfUncached) + .await; + let model_info = models_manager + .get_model_info(model, &config.to_models_manager_config()) + .await; + assert_eq!(model_info.auto_review_model_override, Some(true)); + + core_test_support::submit_thread_settings( + &codex, + codex_protocol::protocol::ThreadSettingsOverrides { + model: Some(model.to_string()), + ..Default::default() + }, + ) + .await?; + + let cwd_path = cwd.path().to_path_buf(); + let (sandbox_policy, permission_profile) = + turn_permission_fields(PermissionProfile::read_only(), cwd_path.as_path()); + codex + .submit(Op::UserInput { + items: vec![UserInput::Text { + text: "run the Guardian model override check".into(), + text_elements: Vec::new(), + }], + environments: None, + final_output_json_schema: None, + responsesapi_client_metadata: None, + thread_settings: codex_protocol::protocol::ThreadSettingsOverrides { + cwd: Some(cwd_path), + approval_policy: Some(AskForApproval::OnRequest), + sandbox_policy: Some(sandbox_policy), + permission_profile, + ..Default::default() + }, + }) + .await?; + + let permissions_request = wait_for_event(&codex, |event| { + matches!( + event, + EventMsg::RequestPermissions(_) | EventMsg::TurnComplete(_) + ) + }) + .await; + let EventMsg::RequestPermissions(permissions_request) = permissions_request else { + panic!("expected request_permissions before completion"); + }; + assert_eq!(permissions_request.call_id, permissions_call_id); + codex + .submit(Op::RequestPermissionsResponse { + id: permissions_request.call_id, + response: RequestPermissionsResponse { + permissions: permissions_request.permissions, + scope: PermissionGrantScope::Turn, + strict_auto_review: true, + }, + }) + .await?; + + wait_for_event(&codex, |event| matches!(event, EventMsg::TurnComplete(_))).await; + + let requests = responses.requests(); + assert_eq!(requests.len(), 4); + assert_eq!(requests[2].body_json()["model"].as_str(), Some(model)); + + Ok(()) +} + +fn remote_model_with_auto_review_override(slug: &str) -> ModelInfo { + ModelInfo { + slug: slug.to_string(), + display_name: format!("{slug} display"), + description: Some(format!("{slug} description")), + default_reasoning_level: Some(ReasoningEffort::Medium), + supported_reasoning_levels: vec![ReasoningEffortPreset { + effort: ReasoningEffort::Medium, + description: ReasoningEffort::Medium.to_string(), + }], + shell_type: ConfigShellToolType::ShellCommand, + visibility: ModelVisibility::List, + supported_in_api: true, + input_modalities: default_input_modalities(), + used_fallback_model_metadata: false, + supports_search_tool: false, + auto_review_model_override: Some(true), + priority: 1, + additional_speed_tiers: Vec::new(), + service_tiers: Vec::new(), + upgrade: None, + base_instructions: "base instructions".to_string(), + model_messages: None, + supports_reasoning_summaries: false, + default_reasoning_summary: ReasoningSummary::Auto, + support_verbosity: false, + default_verbosity: None, + availability_nux: None, + apply_patch_tool_type: None, + web_search_tool_type: Default::default(), + truncation_policy: TruncationPolicyConfig::bytes(/*limit*/ 10_000), + supports_parallel_tool_calls: false, + supports_image_detail_original: false, + context_window: Some(272_000), + max_context_window: None, + auto_compact_token_limit: None, + effective_context_window_percent: 95, + experimental_supported_tools: Vec::new(), + } +} diff --git a/codex-rs/core/tests/suite/mod.rs b/codex-rs/core/tests/suite/mod.rs index 87772aa279..30553314dd 100644 --- a/codex-rs/core/tests/suite/mod.rs +++ b/codex-rs/core/tests/suite/mod.rs @@ -35,6 +35,7 @@ mod agents_md; mod apply_patch_cli; #[cfg(not(target_os = "windows"))] mod approvals; +mod auto_review; mod cli_stream; mod client; mod client_websockets; diff --git a/codex-rs/core/tests/suite/remote_models.rs b/codex-rs/core/tests/suite/remote_models.rs index e22c6f98ca..236e0ed3ec 100644 --- a/codex-rs/core/tests/suite/remote_models.rs +++ b/codex-rs/core/tests/suite/remote_models.rs @@ -1,14 +1,12 @@ #![cfg(not(target_os = "windows"))] #![allow(clippy::expect_used)] use anyhow::Result; -use codex_features::Feature; use codex_login::CodexAuth; use codex_model_provider_info::ModelProviderInfo; use codex_model_provider_info::built_in_model_providers; use codex_models_manager::bundled_models_response; use codex_models_manager::manager::RefreshStrategy; use codex_models_manager::manager::SharedModelsManager; -use codex_protocol::config_types::ApprovalsReviewer; use codex_protocol::config_types::ReasoningSummary; use codex_protocol::models::PermissionProfile; use codex_protocol::openai_models::ConfigShellToolType; @@ -24,8 +22,6 @@ use codex_protocol::protocol::AskForApproval; use codex_protocol::protocol::EventMsg; use codex_protocol::protocol::ExecCommandSource; use codex_protocol::protocol::Op; -use codex_protocol::request_permissions::PermissionGrantScope; -use codex_protocol::request_permissions::RequestPermissionsResponse; use codex_protocol::user_input::UserInput; use core_test_support::load_default_config_for_test; use core_test_support::responses::ev_assistant_message; @@ -603,172 +599,6 @@ async fn remote_models_remote_model_uses_unified_exec() -> Result<()> { Ok(()) } -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn remote_models_auto_review_override_uses_parent_model_for_guardian() -> Result<()> { - skip_if_no_network!(Ok(())); - skip_if_sandbox!(Ok(())); - - let server = MockServer::start().await; - let model = "remote-auto-review-parent"; - let mut remote_model = test_remote_model(model, ModelVisibility::List, /*priority*/ 1); - remote_model.auto_review_model_override = Some(true); - mount_models_once( - &server, - ModelsResponse { - models: vec![remote_model], - }, - ) - .await; - - let permissions_call_id = "auto-review-permissions-call"; - let permissions_args = json!({ - "reason": "exercise strict Guardian model selection", - "permissions": { - "network": { - "enabled": true, - }, - }, - }); - let shell_call_id = "auto-review-shell-call"; - let shell_args = json!({ - "command": "/bin/echo auto-review model override", - "timeout_ms": 5_000, - }); - let responses = mount_sse_sequence( - &server, - vec![ - sse(vec![ - ev_response_created("resp-parent-1"), - ev_function_call( - permissions_call_id, - "request_permissions", - &serde_json::to_string(&permissions_args)?, - ), - ev_completed("resp-parent-1"), - ]), - sse(vec![ - ev_response_created("resp-parent-2"), - ev_function_call( - shell_call_id, - "shell_command", - &serde_json::to_string(&shell_args)?, - ), - ev_completed("resp-parent-2"), - ]), - sse(vec![ - ev_response_created("resp-guardian"), - ev_assistant_message( - "msg-guardian", - &json!({ - "risk_level": "low", - "user_authorization": "high", - "outcome": "allow", - "rationale": "The command only exercises Guardian model selection.", - }) - .to_string(), - ), - ev_completed("resp-guardian"), - ]), - sse(vec![ - ev_response_created("resp-parent-3"), - ev_assistant_message("msg-parent-3", "done"), - ev_completed("resp-parent-3"), - ]), - ], - ) - .await; - - let mut builder = test_codex() - .with_auth(CodexAuth::create_dummy_chatgpt_auth_for_testing()) - .with_config(|config| { - config.model = Some("gpt-5.4".to_string()); - config.approvals_reviewer = ApprovalsReviewer::User; - config - .features - .enable(Feature::ExecPermissionApprovals) - .expect("test config should allow feature update"); - config - .features - .enable(Feature::RequestPermissionsTool) - .expect("test config should allow feature update"); - }); - let TestCodex { - codex, - cwd, - config, - thread_manager, - .. - } = builder.build(&server).await?; - - let models_manager = thread_manager.get_models_manager(); - wait_for_model_available(&models_manager, model).await; - let model_info = models_manager - .get_model_info(model, &config.to_models_manager_config()) - .await; - assert_eq!(model_info.auto_review_model_override, Some(true)); - - core_test_support::submit_thread_settings( - &codex, - codex_protocol::protocol::ThreadSettingsOverrides { - model: Some(model.to_string()), - ..Default::default() - }, - ) - .await?; - - let cwd_path = cwd.path().to_path_buf(); - let (sandbox_policy, permission_profile) = - turn_permission_fields(PermissionProfile::read_only(), cwd_path.as_path()); - codex - .submit(Op::UserInput { - items: vec![UserInput::Text { - text: "run the Guardian model override check".into(), - text_elements: Vec::new(), - }], - environments: None, - final_output_json_schema: None, - responsesapi_client_metadata: None, - thread_settings: codex_protocol::protocol::ThreadSettingsOverrides { - cwd: Some(cwd_path), - approval_policy: Some(AskForApproval::OnRequest), - sandbox_policy: Some(sandbox_policy), - permission_profile, - ..Default::default() - }, - }) - .await?; - - let permissions_request = wait_for_event(&codex, |event| { - matches!( - event, - EventMsg::RequestPermissions(_) | EventMsg::TurnComplete(_) - ) - }) - .await; - let EventMsg::RequestPermissions(permissions_request) = permissions_request else { - panic!("expected request_permissions before completion"); - }; - assert_eq!(permissions_request.call_id, permissions_call_id); - codex - .submit(Op::RequestPermissionsResponse { - id: permissions_request.call_id, - response: RequestPermissionsResponse { - permissions: permissions_request.permissions, - scope: PermissionGrantScope::Turn, - strict_auto_review: true, - }, - }) - .await?; - - wait_for_event(&codex, |event| matches!(event, EventMsg::TurnComplete(_))).await; - - let requests = responses.requests(); - assert_eq!(requests.len(), 4); - assert_eq!(requests[2].body_json()["model"].as_str(), Some(model)); - - Ok(()) -} - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn remote_models_truncation_policy_without_override_preserves_remote() -> Result<()> { skip_if_no_network!(Ok(()));