mirror of
https://github.com/openai/codex.git
synced 2026-04-26 15:45:02 +00:00
feat: show runtime metrics in console (#10278)
Summary of changes:
- Adds a new feature flag: runtime_metrics
- Declared in core/src/features.rs
- Added to core/config.schema.json
- Wired into OTEL init in core/src/otel_init.rs
- Enables on-demand runtime metric snapshots in OTEL
- Adds runtime_metrics: bool to otel/src/config.rs
- Enables experimental custom reader features in otel/Cargo.toml
- Adds snapshot/reset/summary APIs in:
- otel/src/lib.rs
- otel/src/metrics/client.rs
- otel/src/metrics/config.rs
- otel/src/metrics/error.rs
- Defines metric names and a runtime summary builder
- New files:
- otel/src/metrics/names.rs
- otel/src/metrics/runtime_metrics.rs
- Summarizes totals for:
- Tool calls
- API requests
- SSE/streaming events
- Instruments metrics collection in OTEL manager
- otel/src/traces/otel_manager.rs now records:
- API call counts + durations
- SSE event counts + durations (success/failure)
- Tool call metrics now use shared constants
- Surfaces runtime metrics in the TUI
- Resets runtime metrics at turn start in tui/src/chatwidget.rs
- Displays metrics in the final separator line in
tui/src/history_cell.rs
- Adds tests
- New OTEL tests:
- otel/tests/suite/snapshot.rs
- otel/tests/suite/runtime_summary.rs
- New TUI test:
- final_message_separator_includes_runtime_metrics in
tui/src/history_cell.rs
Scope:
- 19 files changed
- ~652 insertions, 38 deletions
<img width="922" height="169" alt="Screenshot 2026-01-30 at 4 11 34 PM"
src="https://github.com/user-attachments/assets/1efd754d-a16d-4564-83a5-f4442fd2f998"
/>
This commit is contained in:
@@ -1,5 +1,7 @@
|
||||
mod manager_metrics;
|
||||
mod otlp_http_loopback;
|
||||
mod runtime_summary;
|
||||
mod send;
|
||||
mod snapshot;
|
||||
mod timing;
|
||||
mod validation;
|
||||
|
||||
77
codex-rs/otel/tests/suite/runtime_summary.rs
Normal file
77
codex-rs/otel/tests/suite/runtime_summary.rs
Normal file
@@ -0,0 +1,77 @@
|
||||
use codex_app_server_protocol::AuthMode;
|
||||
use codex_otel::OtelManager;
|
||||
use codex_otel::RuntimeMetricTotals;
|
||||
use codex_otel::RuntimeMetricsSummary;
|
||||
use codex_otel::metrics::MetricsClient;
|
||||
use codex_otel::metrics::MetricsConfig;
|
||||
use codex_otel::metrics::Result;
|
||||
use codex_protocol::ThreadId;
|
||||
use codex_protocol::protocol::SessionSource;
|
||||
use eventsource_stream::Event as StreamEvent;
|
||||
use opentelemetry_sdk::metrics::InMemoryMetricExporter;
|
||||
use pretty_assertions::assert_eq;
|
||||
use std::time::Duration;
|
||||
|
||||
#[test]
|
||||
fn runtime_metrics_summary_collects_tool_api_and_streaming_metrics() -> Result<()> {
|
||||
let exporter = InMemoryMetricExporter::default();
|
||||
let metrics = MetricsClient::new(
|
||||
MetricsConfig::in_memory("test", "codex-cli", env!("CARGO_PKG_VERSION"), exporter)
|
||||
.with_runtime_reader(),
|
||||
)?;
|
||||
let manager = OtelManager::new(
|
||||
ThreadId::new(),
|
||||
"gpt-5.1",
|
||||
"gpt-5.1",
|
||||
Some("account-id".to_string()),
|
||||
None,
|
||||
Some(AuthMode::ApiKey),
|
||||
true,
|
||||
"tty".to_string(),
|
||||
SessionSource::Cli,
|
||||
)
|
||||
.with_metrics(metrics);
|
||||
|
||||
manager.reset_runtime_metrics();
|
||||
|
||||
manager.tool_result(
|
||||
"shell",
|
||||
"call-1",
|
||||
"{\"cmd\":\"echo\"}",
|
||||
Duration::from_millis(250),
|
||||
true,
|
||||
"ok",
|
||||
);
|
||||
manager.record_api_request(1, Some(200), None, Duration::from_millis(300));
|
||||
let sse_response: std::result::Result<
|
||||
Option<std::result::Result<StreamEvent, eventsource_stream::EventStreamError<&str>>>,
|
||||
tokio::time::error::Elapsed,
|
||||
> = Ok(Some(Ok(StreamEvent {
|
||||
event: "response.created".to_string(),
|
||||
data: "{}".to_string(),
|
||||
id: String::new(),
|
||||
retry: None,
|
||||
})));
|
||||
manager.log_sse_event(&sse_response, Duration::from_millis(120));
|
||||
|
||||
let summary = manager
|
||||
.runtime_metrics_summary()
|
||||
.expect("runtime metrics summary should be available");
|
||||
let expected = RuntimeMetricsSummary {
|
||||
tool_calls: RuntimeMetricTotals {
|
||||
count: 1,
|
||||
duration_ms: 250,
|
||||
},
|
||||
api_calls: RuntimeMetricTotals {
|
||||
count: 1,
|
||||
duration_ms: 300,
|
||||
},
|
||||
streaming_events: RuntimeMetricTotals {
|
||||
count: 1,
|
||||
duration_ms: 120,
|
||||
},
|
||||
};
|
||||
assert_eq!(summary, expected);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
120
codex-rs/otel/tests/suite/snapshot.rs
Normal file
120
codex-rs/otel/tests/suite/snapshot.rs
Normal file
@@ -0,0 +1,120 @@
|
||||
use crate::harness::attributes_to_map;
|
||||
use crate::harness::find_metric;
|
||||
use codex_app_server_protocol::AuthMode;
|
||||
use codex_otel::OtelManager;
|
||||
use codex_otel::metrics::MetricsClient;
|
||||
use codex_otel::metrics::MetricsConfig;
|
||||
use codex_otel::metrics::Result;
|
||||
use codex_protocol::ThreadId;
|
||||
use codex_protocol::protocol::SessionSource;
|
||||
use opentelemetry_sdk::metrics::InMemoryMetricExporter;
|
||||
use opentelemetry_sdk::metrics::data::AggregatedMetrics;
|
||||
use opentelemetry_sdk::metrics::data::MetricData;
|
||||
use pretty_assertions::assert_eq;
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
#[test]
|
||||
fn snapshot_collects_metrics_without_shutdown() -> Result<()> {
|
||||
let exporter = InMemoryMetricExporter::default();
|
||||
let config = MetricsConfig::in_memory(
|
||||
"test",
|
||||
"codex-cli",
|
||||
env!("CARGO_PKG_VERSION"),
|
||||
exporter.clone(),
|
||||
)
|
||||
.with_tag("service", "codex-cli")?
|
||||
.with_runtime_reader();
|
||||
let metrics = MetricsClient::new(config)?;
|
||||
|
||||
metrics.counter(
|
||||
"codex.tool.call",
|
||||
1,
|
||||
&[("tool", "shell"), ("success", "true")],
|
||||
)?;
|
||||
|
||||
let snapshot = metrics.snapshot()?;
|
||||
|
||||
let metric = find_metric(&snapshot, "codex.tool.call").expect("counter metric missing");
|
||||
let attrs = match metric.data() {
|
||||
AggregatedMetrics::U64(data) => match data {
|
||||
MetricData::Sum(sum) => {
|
||||
let points: Vec<_> = sum.data_points().collect();
|
||||
assert_eq!(points.len(), 1);
|
||||
attributes_to_map(points[0].attributes())
|
||||
}
|
||||
_ => panic!("unexpected counter aggregation"),
|
||||
},
|
||||
_ => panic!("unexpected counter data type"),
|
||||
};
|
||||
|
||||
let expected = BTreeMap::from([
|
||||
("service".to_string(), "codex-cli".to_string()),
|
||||
("success".to_string(), "true".to_string()),
|
||||
("tool".to_string(), "shell".to_string()),
|
||||
]);
|
||||
assert_eq!(attrs, expected);
|
||||
|
||||
let finished = exporter
|
||||
.get_finished_metrics()
|
||||
.expect("finished metrics should be readable");
|
||||
assert!(finished.is_empty(), "expected no periodic exports yet");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn manager_snapshot_metrics_collects_without_shutdown() -> Result<()> {
|
||||
let exporter = InMemoryMetricExporter::default();
|
||||
let config = MetricsConfig::in_memory("test", "codex-cli", env!("CARGO_PKG_VERSION"), exporter)
|
||||
.with_tag("service", "codex-cli")?
|
||||
.with_runtime_reader();
|
||||
let metrics = MetricsClient::new(config)?;
|
||||
let manager = OtelManager::new(
|
||||
ThreadId::new(),
|
||||
"gpt-5.1",
|
||||
"gpt-5.1",
|
||||
Some("account-id".to_string()),
|
||||
None,
|
||||
Some(AuthMode::ApiKey),
|
||||
true,
|
||||
"tty".to_string(),
|
||||
SessionSource::Cli,
|
||||
)
|
||||
.with_metrics(metrics);
|
||||
|
||||
manager.counter(
|
||||
"codex.tool.call",
|
||||
1,
|
||||
&[("tool", "shell"), ("success", "true")],
|
||||
);
|
||||
|
||||
let snapshot = manager.snapshot_metrics()?;
|
||||
let metric = find_metric(&snapshot, "codex.tool.call").expect("counter metric missing");
|
||||
let attrs = match metric.data() {
|
||||
AggregatedMetrics::U64(data) => match data {
|
||||
MetricData::Sum(sum) => {
|
||||
let points: Vec<_> = sum.data_points().collect();
|
||||
assert_eq!(points.len(), 1);
|
||||
attributes_to_map(points[0].attributes())
|
||||
}
|
||||
_ => panic!("unexpected counter aggregation"),
|
||||
},
|
||||
_ => panic!("unexpected counter data type"),
|
||||
};
|
||||
|
||||
let expected = BTreeMap::from([
|
||||
(
|
||||
"app.version".to_string(),
|
||||
env!("CARGO_PKG_VERSION").to_string(),
|
||||
),
|
||||
("auth_mode".to_string(), AuthMode::ApiKey.to_string()),
|
||||
("model".to_string(), "gpt-5.1".to_string()),
|
||||
("service".to_string(), "codex-cli".to_string()),
|
||||
("session_source".to_string(), "cli".to_string()),
|
||||
("success".to_string(), "true".to_string()),
|
||||
("tool".to_string(), "shell".to_string()),
|
||||
]);
|
||||
assert_eq!(attrs, expected);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
Reference in New Issue
Block a user