Compare commits

...

24 Commits

Author SHA1 Message Date
Ahmed Ibrahim
e6016f5489 client 2025-12-02 16:16:38 -08:00
Ahmed Ibrahim
2721498ec9 client 2025-12-02 14:54:33 -08:00
Ahmed Ibrahim
47ef2cd9ca client 2025-12-02 14:45:44 -08:00
Ahmed Ibrahim
db70faab42 copy 2025-11-25 16:01:19 -08:00
Ahmed Ibrahim
1e7796570a copy 2025-11-25 15:56:48 -08:00
Ahmed Ibrahim
11b13914a8 copy 2025-11-25 15:54:53 -08:00
Ahmed Ibrahim
567844fe05 codex-status 2025-11-25 15:47:39 -08:00
Ahmed Ibrahim
6eb0474455 Merge branch 'codex-status' of https://github.com/openai/codex into codex-status 2025-11-25 15:42:32 -08:00
Ahmed Ibrahim
af1254fc4e codex-status 2025-11-25 15:39:03 -08:00
Ahmed Ibrahim
764aff6753 codex-status 2025-11-25 15:38:54 -08:00
Ahmed Ibrahim
e2a55921ec Merge branch 'main' into codex-status 2025-11-25 15:36:46 -08:00
Ahmed Ibrahim
08b6d9ef1f codex-status 2025-11-25 15:35:43 -08:00
Ahmed Ibrahim
bbf536847c codex-status 2025-11-25 15:34:32 -08:00
Ahmed Ibrahim
0d0779d08a codex-status 2025-11-25 15:32:46 -08:00
Ahmed Ibrahim
087e571198 tests 2025-11-25 15:17:14 -08:00
Ahmed Ibrahim
70b613be81 warning 2025-11-25 14:18:41 -08:00
Ahmed Ibrahim
53ff941cf3 warning 2025-11-25 14:18:25 -08:00
Ahmed Ibrahim
6b66534356 status 2025-11-25 12:49:59 -08:00
Ahmed Ibrahim
1938116a5d status 2025-11-25 12:49:04 -08:00
Ahmed Ibrahim
075d50677d use client 2025-11-25 12:44:43 -08:00
Ahmed Ibrahim
5c40534e98 use client 2025-11-25 12:42:25 -08:00
Ahmed Ibrahim
f05492fc94 crate 2025-11-25 12:35:40 -08:00
Ahmed Ibrahim
44ff9fcb69 review 2025-11-25 12:35:40 -08:00
Ahmed Ibrahim
b4a1a500ec status 2025-11-25 12:35:40 -08:00
9 changed files with 511 additions and 9 deletions

1
codex-rs/Cargo.lock generated
View File

@@ -1144,6 +1144,7 @@ dependencies = [
"codex-apply-patch",
"codex-arg0",
"codex-async-utils",
"codex-client",
"codex-core",
"codex-execpolicy",
"codex-file-search",

View File

@@ -1,8 +1,8 @@
[package]
name = "codex-core"
version.workspace = true
edition.workspace = true
license.workspace = true
name = "codex-core"
version.workspace = true
[lib]
doctest = false
@@ -18,12 +18,13 @@ askama = { workspace = true }
async-channel = { workspace = true }
async-trait = { workspace = true }
base64 = { workspace = true }
chrono = { workspace = true, features = ["serde"] }
chardetng = { workspace = true }
chrono = { workspace = true, features = ["serde"] }
codex-api = { workspace = true }
codex-app-server-protocol = { workspace = true }
codex-apply-patch = { workspace = true }
codex-async-utils = { workspace = true }
codex-api = { workspace = true }
codex-client = { workspace = true }
codex-execpolicy = { workspace = true }
codex-file-search = { workspace = true }
codex-git = { workspace = true }
@@ -37,8 +38,8 @@ codex-utils-string = { workspace = true }
codex-windows-sandbox = { package = "codex-windows-sandbox", path = "../windows-sandbox-rs" }
dirs = { workspace = true }
dunce = { workspace = true }
env-flags = { workspace = true }
encoding_rs = { workspace = true }
env-flags = { workspace = true }
eventsource-stream = { workspace = true }
futures = { workspace = true }
http = { workspace = true }
@@ -46,8 +47,10 @@ indexmap = { workspace = true }
keyring = { workspace = true, features = ["crypto-rust"] }
libc = { workspace = true }
mcp-types = { workspace = true }
once_cell = { workspace = true }
os_info = { workspace = true }
rand = { workspace = true }
regex = { workspace = true }
regex-lite = { workspace = true }
reqwest = { workspace = true, features = ["json", "stream"] }
serde = { workspace = true, features = ["derive"] }
@@ -57,9 +60,6 @@ sha2 = { workspace = true }
shlex = { workspace = true }
similar = { workspace = true }
strum_macros = { workspace = true }
url = { workspace = true }
once_cell = { workspace = true }
regex = { workspace = true }
tempfile = { workspace = true }
test-case = "3.3.1"
test-log = { workspace = true }
@@ -83,6 +83,7 @@ toml_edit = { workspace = true }
tracing = { workspace = true, features = ["log"] }
tree-sitter = { workspace = true }
tree-sitter-bash = { workspace = true }
url = { workspace = true }
uuid = { workspace = true, features = ["serde", "v4", "v5"] }
which = { workspace = true }
wildmatch = { workspace = true }
@@ -92,9 +93,9 @@ deterministic_process_ids = []
[target.'cfg(target_os = "linux")'.dependencies]
keyring = { workspace = true, features = ["linux-native-async-persistent"] }
landlock = { workspace = true }
seccompiler = { workspace = true }
keyring = { workspace = true, features = ["linux-native-async-persistent"] }
[target.'cfg(target_os = "macos")'.dependencies]
core-foundation = "0.9"

View File

@@ -106,6 +106,8 @@ use crate::shell;
use crate::state::ActiveTurn;
use crate::state::SessionServices;
use crate::state::SessionState;
use crate::status::ComponentHealth;
use crate::status::maybe_codex_status_warning;
use crate::tasks::GhostSnapshotTask;
use crate::tasks::ReviewTask;
use crate::tasks::SessionTask;
@@ -451,6 +453,16 @@ impl Session {
}
}
pub(crate) async fn replace_codex_backend_status(
&self,
status: ComponentHealth,
) -> Option<ComponentHealth> {
let mut guard = self.services.codex_backend_status.lock().await;
let previous = *guard;
*guard = Some(status);
previous
}
async fn new(
session_configuration: SessionConfiguration,
config: Arc<Config>,
@@ -567,6 +579,7 @@ impl Session {
auth_manager: Arc::clone(&auth_manager),
otel_event_manager,
tool_approvals: Mutex::new(ApprovalStore::default()),
codex_backend_status: Mutex::new(None),
};
let sess = Arc::new(Session {
@@ -2180,6 +2193,19 @@ async fn try_run_turn(
});
sess.persist_rollout_items(&[rollout_item]).await;
let sess_clone = Arc::clone(&sess);
let turn_context_clone = Arc::clone(&turn_context);
tokio::spawn(async move {
if let Some(message) = maybe_codex_status_warning(sess_clone.as_ref()).await {
sess_clone
.send_event(
&turn_context_clone,
EventMsg::Warning(WarningEvent { message }),
)
.await;
}
});
let mut stream = turn_context
.client
.clone()
@@ -2694,6 +2720,7 @@ mod tests {
auth_manager: Arc::clone(&auth_manager),
otel_event_manager: otel_event_manager.clone(),
tool_approvals: Mutex::new(ApprovalStore::default()),
codex_backend_status: Mutex::new(None),
};
let turn_context = Session::make_turn_context(
@@ -2772,6 +2799,7 @@ mod tests {
auth_manager: Arc::clone(&auth_manager),
otel_event_manager: otel_event_manager.clone(),
tool_approvals: Mutex::new(ApprovalStore::default()),
codex_backend_status: Mutex::new(None),
};
let turn_context = Arc::new(Session::make_turn_context(

View File

@@ -42,6 +42,7 @@ pub mod parse_command;
pub mod powershell;
mod response_processing;
pub mod sandboxing;
pub mod status;
mod text_encoding;
pub mod token_data;
mod truncate;

View File

@@ -3,6 +3,7 @@ use std::sync::Arc;
use crate::AuthManager;
use crate::RolloutRecorder;
use crate::mcp_connection_manager::McpConnectionManager;
use crate::status::ComponentHealth;
use crate::tools::sandboxing::ApprovalStore;
use crate::unified_exec::UnifiedExecSessionManager;
use crate::user_notification::UserNotifier;
@@ -22,4 +23,5 @@ pub(crate) struct SessionServices {
pub(crate) auth_manager: Arc<AuthManager>,
pub(crate) otel_event_manager: OtelEventManager,
pub(crate) tool_approvals: Mutex<ApprovalStore>,
pub(crate) codex_backend_status: Mutex<Option<ComponentHealth>>,
}

257
codex-rs/core/src/status.rs Normal file
View File

@@ -0,0 +1,257 @@
use std::sync::OnceLock;
use std::time::Duration;
use crate::codex::Session;
use anyhow::Context;
use anyhow::Result;
use anyhow::anyhow;
use anyhow::bail;
use codex_client::HttpTransport;
use codex_client::Request;
use codex_client::ReqwestTransport;
use codex_client::RetryOn;
use codex_client::RetryPolicy;
use codex_client::run_with_retry;
use http::header::CONTENT_TYPE;
use reqwest::Method;
use serde::Deserialize;
use serde::Serialize;
use strum_macros::Display;
const STATUS_WIDGET_URL: &str = "https://status.openai.com/proxy/status.openai.com";
const CODEX_COMPONENT_NAME: &str = "Codex";
static TEST_STATUS_WIDGET_URL: OnceLock<String> = OnceLock::new();
#[derive(Debug, Clone, Display, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub(crate) enum ComponentHealth {
#[strum(to_string = "operational")]
Operational,
#[strum(to_string = "degraded performance")]
DegradedPerformance,
#[strum(to_string = "partial outage")]
PartialOutage,
#[strum(to_string = "major outage")]
MajorOutage,
#[strum(to_string = "under maintenance")]
UnderMaintenance,
#[serde(other)]
#[strum(to_string = "unknown")]
Unknown,
}
impl ComponentHealth {
fn operational() -> Self {
Self::Operational
}
pub(crate) fn is_operational(self) -> bool {
self == Self::Operational
}
}
pub(crate) async fn maybe_codex_status_warning(session: &Session) -> Option<String> {
let Ok(status) = fetch_codex_health().await else {
return None;
};
let previous = session.replace_codex_backend_status(status).await;
if status.is_operational() || previous == Some(status) {
return None;
}
Some(format!(
"Codex is experiencing a {status}. If a response stalls, try again later. You can follow incident updates at status.openai.com."
))
}
async fn fetch_codex_health() -> Result<ComponentHealth> {
let status_widget_url = status_widget_url();
let client = reqwest::Client::builder()
.connect_timeout(Duration::from_millis(200))
.timeout(Duration::from_millis(300))
.build()
.context("building HTTP client")?;
let transport = ReqwestTransport::new(client);
let policy = RetryPolicy {
max_attempts: 0,
base_delay: Duration::from_millis(100),
retry_on: RetryOn {
retry_429: true,
retry_5xx: true,
retry_transport: true,
},
};
let response = run_with_retry(
policy,
|| Request::new(Method::GET, status_widget_url.clone()),
|req, _attempt| {
let transport = transport.clone();
async move { transport.execute(req).await }
},
)
.await
.context("requesting status widget")?;
let content_type = response
.headers
.get(CONTENT_TYPE)
.and_then(|value| value.to_str().ok())
.unwrap_or_default()
.to_ascii_lowercase();
if !content_type.contains("json") {
let snippet = String::from_utf8_lossy(&response.body)
.chars()
.take(200)
.collect::<String>();
bail!(
"Expected JSON from {status_widget_url}: Content-Type={content_type}. Body starts with: {snippet:?}"
);
}
let payload: StatusPayload =
serde_json::from_slice(&response.body).context("parsing status widget JSON")?;
derive_component_health(&payload, CODEX_COMPONENT_NAME)
}
#[derive(Debug, Clone, Deserialize, Default)]
struct StatusPayload {
#[serde(default)]
summary: Summary,
}
#[derive(Debug, Clone, Deserialize, Default)]
struct Summary {
#[serde(default)]
components: Vec<Component>,
#[serde(default)]
affected_components: Vec<AffectedComponent>,
}
#[derive(Debug, Clone, Deserialize)]
struct Component {
id: String,
name: String,
}
#[derive(Debug, Clone, Deserialize)]
struct AffectedComponent {
component_id: String,
#[serde(default = "ComponentHealth::operational")]
status: ComponentHealth,
}
fn derive_component_health(
payload: &StatusPayload,
component_name: &str,
) -> Result<ComponentHealth> {
let component = payload
.summary
.components
.iter()
.find(|component| component.name == component_name)
.ok_or_else(|| anyhow!("Component {component_name:?} not found in status summary"))?;
let status = payload
.summary
.affected_components
.iter()
.find(|affected| affected.component_id == component.id)
.map(|affected| affected.status)
.unwrap_or(ComponentHealth::Operational);
Ok(status)
}
fn status_widget_url() -> String {
TEST_STATUS_WIDGET_URL
.get()
.cloned()
.unwrap_or_else(|| STATUS_WIDGET_URL.to_string())
}
#[doc(hidden)]
#[cfg_attr(not(test), allow(dead_code))]
pub fn set_test_status_widget_url(url: impl Into<String>) {
let _ = TEST_STATUS_WIDGET_URL.set(url.into());
}
#[cfg(test)]
mod tests {
use super::*;
use pretty_assertions::assert_eq;
use serde_json::json;
#[test]
fn uses_affected_component_status() {
let payload = serde_json::from_value::<StatusPayload>(json!({
"summary": {
"id": "sum-1",
"name": "OpenAI",
"components": [
{"id": "cmp-1", "name": "Codex", "status_page_id": "page-1"}
],
"affected_components": [
{"component_id": "cmp-1", "status": "major_outage"}
]
}
}))
.expect("valid payload");
let status = derive_component_health(&payload, "Codex").expect("codex component exists");
assert_eq!(status, ComponentHealth::MajorOutage);
assert!(!status.is_operational());
}
#[test]
fn unknown_status_is_preserved_as_unknown() {
let payload = serde_json::from_value::<StatusPayload>(json!({
"summary": {
"id": "sum-1",
"name": "OpenAI",
"components": [
{"id": "cmp-1", "name": "Codex", "status_page_id": "page-1"}
],
"affected_components": [
{"component_id": "cmp-1", "status": "custom_status"}
]
}
}))
.expect("valid payload");
let status = derive_component_health(&payload, "Codex").expect("codex component exists");
assert_eq!(status, ComponentHealth::Unknown);
assert!(!status.is_operational());
}
#[test]
fn missing_component_returns_error() {
let payload = serde_json::from_value::<StatusPayload>(json!({
"summary": {
"id": "sum-1",
"name": "OpenAI",
"components": [],
"affected_components": []
}
}))
.expect("valid payload");
let error =
derive_component_health(&payload, "Codex").expect_err("missing component should error");
assert!(
error
.to_string()
.contains("Component \"Codex\" not found in status summary")
);
}
}

View File

@@ -1,5 +1,6 @@
use std::sync::Arc;
use std::sync::Mutex;
use std::time::Duration;
use anyhow::Result;
use base64::Engine;
@@ -546,6 +547,19 @@ pub async fn mount_sse_once(server: &MockServer, body: String) -> ResponseMock {
response_mock
}
pub async fn mount_sse_once_with_delay(
server: &MockServer,
body: String,
delay: Duration,
) -> ResponseMock {
let (mock, response_mock) = base_mock();
mock.respond_with(sse_response(body).set_delay(delay))
.up_to_n_times(1)
.mount(server)
.await;
response_mock
}
pub async fn mount_compact_json_once_match<M>(
server: &MockServer,
matcher: M,

View File

@@ -0,0 +1,197 @@
#![allow(clippy::unwrap_used, clippy::expect_used)]
use std::sync::Arc;
use std::sync::atomic::AtomicUsize;
use std::sync::atomic::Ordering;
use std::time::Duration;
use codex_core::protocol::EventMsg;
use codex_core::protocol::Op;
use codex_core::protocol::WarningEvent;
use codex_core::status::set_test_status_widget_url;
use codex_protocol::user_input::UserInput;
use core_test_support::responses::ev_assistant_message;
use core_test_support::responses::ev_completed;
use core_test_support::responses::ev_response_created;
use core_test_support::responses::mount_sse_once_with_delay;
use core_test_support::responses::sse;
use core_test_support::responses::start_mock_server;
use core_test_support::test_codex::test_codex;
use core_test_support::wait_for_event;
use pretty_assertions::assert_eq;
use wiremock::Mock;
use wiremock::Request;
use wiremock::Respond;
use wiremock::ResponseTemplate;
use wiremock::matchers::method;
use wiremock::matchers::path;
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn emits_warning_when_status_is_degraded_at_turn_start() {
let status_server = start_mock_server().await;
let status_path = "/proxy/status.openai.com";
Mock::given(method("GET"))
.and(path(status_path))
.respond_with(status_payload("major_outage"))
.mount(&status_server)
.await;
set_test_status_widget_url(format!("{}{}", status_server.uri(), status_path));
let responses_server = start_mock_server().await;
let stalled_response = sse(vec![
ev_response_created("resp-1"),
ev_assistant_message("msg-1", "finally"),
ev_completed("resp-1"),
]);
let _responses_mock = mount_sse_once_with_delay(
&responses_server,
stalled_response,
Duration::from_millis(10),
)
.await;
let test_codex = test_codex().build(&responses_server).await.unwrap();
let codex = test_codex.codex;
codex
.submit(Op::UserInput {
items: vec![UserInput::Text { text: "hi".into() }],
})
.await
.unwrap();
let warning = wait_for_event(&codex, |event| matches!(event, EventMsg::Warning(_))).await;
let EventMsg::Warning(WarningEvent { message }) = warning else {
panic!("expected warning event");
};
assert_eq!(
message,
"Codex is experiencing a major outage. If a response stalls, try again later. You can follow incident updates at status.openai.com.",
"unexpected warning message"
);
wait_for_event(&codex, |event| matches!(event, EventMsg::TaskComplete(_))).await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn warns_once_per_status_change_only_when_unhealthy() {
let status_server = start_mock_server().await;
let status_path = "/proxy/status.openai.com";
let responder = SequenceResponder::new(vec!["major_outage", "partial_outage"]);
Mock::given(method("GET"))
.and(path(status_path))
.respond_with(responder)
.mount(&status_server)
.await;
set_test_status_widget_url(format!("{}{}", status_server.uri(), status_path));
let responses_server = start_mock_server().await;
let stalled_response = sse(vec![
ev_response_created("resp-1"),
ev_assistant_message("msg-1", "finally"),
ev_completed("resp-1"),
]);
let _responses_mock = mount_sse_once_with_delay(
&responses_server,
stalled_response,
Duration::from_millis(300),
)
.await;
let test_codex = test_codex().build(&responses_server).await.unwrap();
let codex = test_codex.codex;
codex
.submit(Op::UserInput {
items: vec![UserInput::Text { text: "hi".into() }],
})
.await
.unwrap();
let first_warning = wait_for_event(&codex, |event| matches!(event, EventMsg::Warning(_))).await;
let EventMsg::Warning(WarningEvent { message }) = first_warning else {
panic!("expected warning event");
};
assert_eq!(
message,
"Codex is experiencing a major outage. If a response stalls, try again later. You can follow incident updates at status.openai.com.",
);
wait_for_event(&codex, |event| matches!(event, EventMsg::TaskComplete(_))).await;
codex
.submit(Op::UserInput {
items: vec![UserInput::Text {
text: "second".into(),
}],
})
.await
.unwrap();
let mut saw_warning = false;
let mut saw_complete = false;
while !(saw_warning && saw_complete) {
let event = codex.next_event().await.expect("event");
match event.msg {
EventMsg::Warning(WarningEvent { message }) => {
assert_eq!(
message,
"Codex is experiencing a partial outage. If a response stalls, try again later. You can follow incident updates at status.openai.com.",
);
saw_warning = true;
}
EventMsg::TaskComplete(_) => {
saw_complete = true;
}
_ => {}
}
}
}
fn status_payload(status: &str) -> ResponseTemplate {
ResponseTemplate::new(200)
.insert_header("content-type", "application/json")
.set_body_json(serde_json::json!({
"summary": {
"components": [
{"id": "cmp-1", "name": "Codex", "status_page_id": "page-1"}
],
"affected_components": [
{"component_id": "cmp-1", "status": status}
]
}
}))
}
#[derive(Clone)]
struct SequenceResponder {
statuses: Vec<&'static str>,
calls: Arc<AtomicUsize>,
}
impl SequenceResponder {
fn new(statuses: Vec<&'static str>) -> Self {
Self {
statuses,
calls: Arc::new(AtomicUsize::new(0)),
}
}
}
impl Respond for SequenceResponder {
fn respond(&self, _request: &Request) -> ResponseTemplate {
let call = self.calls.fetch_add(1, Ordering::SeqCst);
let idx = usize::try_from(call).unwrap_or(0);
let status = self
.statuses
.get(idx)
.copied()
.or_else(|| self.statuses.last().copied())
.unwrap_or("operational");
status_payload(status)
}
}

View File

@@ -31,6 +31,7 @@ mod exec;
mod exec_policy;
mod fork_conversation;
mod grep_files;
mod idle_warning;
mod items;
mod json_result;
mod list_dir;