feat: show runtime metrics in console (#10278)

Summary of changes:

- Adds a new feature flag: runtime_metrics
  - Declared in core/src/features.rs
  - Added to core/config.schema.json
  - Wired into OTEL init in core/src/otel_init.rs

- Enables on-demand runtime metric snapshots in OTEL
  - Adds runtime_metrics: bool to otel/src/config.rs
  - Enables experimental custom reader features in otel/Cargo.toml
  - Adds snapshot/reset/summary APIs in:
    - otel/src/lib.rs
    - otel/src/metrics/client.rs
    - otel/src/metrics/config.rs
    - otel/src/metrics/error.rs

- Defines metric names and a runtime summary builder
  - New files:
    - otel/src/metrics/names.rs
    - otel/src/metrics/runtime_metrics.rs
  - Summarizes totals for:
    - Tool calls
    - API requests
    - SSE/streaming events

- Instruments metrics collection in OTEL manager
  - otel/src/traces/otel_manager.rs now records:
    - API call counts + durations
    - SSE event counts + durations (success/failure)
    - Tool call metrics now use shared constants

- Surfaces runtime metrics in the TUI
  - Resets runtime metrics at turn start in tui/src/chatwidget.rs
- Displays metrics in the final separator line in
tui/src/history_cell.rs

- Adds tests
  - New OTEL tests:
    - otel/tests/suite/snapshot.rs
    - otel/tests/suite/runtime_summary.rs
  - New TUI test:
- final_message_separator_includes_runtime_metrics in
tui/src/history_cell.rs

Scope:
- 19 files changed
- ~652 insertions, 38 deletions


<img width="922" height="169" alt="Screenshot 2026-01-30 at 4 11 34 PM"
src="https://github.com/user-attachments/assets/1efd754d-a16d-4564-83a5-f4442fd2f998"
/>
This commit is contained in:
Anton Panasenko
2026-01-30 22:20:02 -08:00
committed by GitHub
parent a8c9e386e7
commit 8660ad6c64
19 changed files with 659 additions and 38 deletions

View File

@@ -1,5 +1,7 @@
mod manager_metrics;
mod otlp_http_loopback;
mod runtime_summary;
mod send;
mod snapshot;
mod timing;
mod validation;

View File

@@ -0,0 +1,77 @@
use codex_app_server_protocol::AuthMode;
use codex_otel::OtelManager;
use codex_otel::RuntimeMetricTotals;
use codex_otel::RuntimeMetricsSummary;
use codex_otel::metrics::MetricsClient;
use codex_otel::metrics::MetricsConfig;
use codex_otel::metrics::Result;
use codex_protocol::ThreadId;
use codex_protocol::protocol::SessionSource;
use eventsource_stream::Event as StreamEvent;
use opentelemetry_sdk::metrics::InMemoryMetricExporter;
use pretty_assertions::assert_eq;
use std::time::Duration;
#[test]
fn runtime_metrics_summary_collects_tool_api_and_streaming_metrics() -> Result<()> {
let exporter = InMemoryMetricExporter::default();
let metrics = MetricsClient::new(
MetricsConfig::in_memory("test", "codex-cli", env!("CARGO_PKG_VERSION"), exporter)
.with_runtime_reader(),
)?;
let manager = OtelManager::new(
ThreadId::new(),
"gpt-5.1",
"gpt-5.1",
Some("account-id".to_string()),
None,
Some(AuthMode::ApiKey),
true,
"tty".to_string(),
SessionSource::Cli,
)
.with_metrics(metrics);
manager.reset_runtime_metrics();
manager.tool_result(
"shell",
"call-1",
"{\"cmd\":\"echo\"}",
Duration::from_millis(250),
true,
"ok",
);
manager.record_api_request(1, Some(200), None, Duration::from_millis(300));
let sse_response: std::result::Result<
Option<std::result::Result<StreamEvent, eventsource_stream::EventStreamError<&str>>>,
tokio::time::error::Elapsed,
> = Ok(Some(Ok(StreamEvent {
event: "response.created".to_string(),
data: "{}".to_string(),
id: String::new(),
retry: None,
})));
manager.log_sse_event(&sse_response, Duration::from_millis(120));
let summary = manager
.runtime_metrics_summary()
.expect("runtime metrics summary should be available");
let expected = RuntimeMetricsSummary {
tool_calls: RuntimeMetricTotals {
count: 1,
duration_ms: 250,
},
api_calls: RuntimeMetricTotals {
count: 1,
duration_ms: 300,
},
streaming_events: RuntimeMetricTotals {
count: 1,
duration_ms: 120,
},
};
assert_eq!(summary, expected);
Ok(())
}

View File

@@ -0,0 +1,120 @@
use crate::harness::attributes_to_map;
use crate::harness::find_metric;
use codex_app_server_protocol::AuthMode;
use codex_otel::OtelManager;
use codex_otel::metrics::MetricsClient;
use codex_otel::metrics::MetricsConfig;
use codex_otel::metrics::Result;
use codex_protocol::ThreadId;
use codex_protocol::protocol::SessionSource;
use opentelemetry_sdk::metrics::InMemoryMetricExporter;
use opentelemetry_sdk::metrics::data::AggregatedMetrics;
use opentelemetry_sdk::metrics::data::MetricData;
use pretty_assertions::assert_eq;
use std::collections::BTreeMap;
#[test]
fn snapshot_collects_metrics_without_shutdown() -> Result<()> {
let exporter = InMemoryMetricExporter::default();
let config = MetricsConfig::in_memory(
"test",
"codex-cli",
env!("CARGO_PKG_VERSION"),
exporter.clone(),
)
.with_tag("service", "codex-cli")?
.with_runtime_reader();
let metrics = MetricsClient::new(config)?;
metrics.counter(
"codex.tool.call",
1,
&[("tool", "shell"), ("success", "true")],
)?;
let snapshot = metrics.snapshot()?;
let metric = find_metric(&snapshot, "codex.tool.call").expect("counter metric missing");
let attrs = match metric.data() {
AggregatedMetrics::U64(data) => match data {
MetricData::Sum(sum) => {
let points: Vec<_> = sum.data_points().collect();
assert_eq!(points.len(), 1);
attributes_to_map(points[0].attributes())
}
_ => panic!("unexpected counter aggregation"),
},
_ => panic!("unexpected counter data type"),
};
let expected = BTreeMap::from([
("service".to_string(), "codex-cli".to_string()),
("success".to_string(), "true".to_string()),
("tool".to_string(), "shell".to_string()),
]);
assert_eq!(attrs, expected);
let finished = exporter
.get_finished_metrics()
.expect("finished metrics should be readable");
assert!(finished.is_empty(), "expected no periodic exports yet");
Ok(())
}
#[test]
fn manager_snapshot_metrics_collects_without_shutdown() -> Result<()> {
let exporter = InMemoryMetricExporter::default();
let config = MetricsConfig::in_memory("test", "codex-cli", env!("CARGO_PKG_VERSION"), exporter)
.with_tag("service", "codex-cli")?
.with_runtime_reader();
let metrics = MetricsClient::new(config)?;
let manager = OtelManager::new(
ThreadId::new(),
"gpt-5.1",
"gpt-5.1",
Some("account-id".to_string()),
None,
Some(AuthMode::ApiKey),
true,
"tty".to_string(),
SessionSource::Cli,
)
.with_metrics(metrics);
manager.counter(
"codex.tool.call",
1,
&[("tool", "shell"), ("success", "true")],
);
let snapshot = manager.snapshot_metrics()?;
let metric = find_metric(&snapshot, "codex.tool.call").expect("counter metric missing");
let attrs = match metric.data() {
AggregatedMetrics::U64(data) => match data {
MetricData::Sum(sum) => {
let points: Vec<_> = sum.data_points().collect();
assert_eq!(points.len(), 1);
attributes_to_map(points[0].attributes())
}
_ => panic!("unexpected counter aggregation"),
},
_ => panic!("unexpected counter data type"),
};
let expected = BTreeMap::from([
(
"app.version".to_string(),
env!("CARGO_PKG_VERSION").to_string(),
),
("auth_mode".to_string(), AuthMode::ApiKey.to_string()),
("model".to_string(), "gpt-5.1".to_string()),
("service".to_string(), "codex-cli".to_string()),
("session_source".to_string(), "cli".to_string()),
("success".to_string(), "true".to_string()),
("tool".to_string(), "shell".to_string()),
]);
assert_eq!(attrs, expected);
Ok(())
}