From 01a8bf0ae3e73d1697d2572a6666d5b47554f87f Mon Sep 17 00:00:00 2001 From: jif-oai Date: Tue, 26 May 2026 15:27:51 +0200 Subject: [PATCH] Add memory tool call metrics to memories extension (#24583) ## Why The memories extension now receives a metrics exporter, but the useful extension-owned signal is the memory tool call itself: which operation ran, which memory area it touched, whether the backend call succeeded, and whether the result was truncated. ## What changed - Added the `codex.memories.tool.call` counter in `ext/memories/src/metrics.rs`. - Emit that counter from `memories/add_ad_hoc_note`, `memories/list`, `memories/read`, and `memories/search` after backend execution. - Tag each call with `tool`, `operation`, `scope`, `status`, and `truncated`. - Pass the existing `MetricsClient` through the memories extension into the tool executors; tests use `None`. ## Verification - `just test -p codex-memories-extension` --- codex-rs/ext/memories/src/extension.rs | 11 +-- codex-rs/ext/memories/src/lib.rs | 1 + codex-rs/ext/memories/src/metrics.rs | 69 +++++++++++++++++++ codex-rs/ext/memories/src/tests.rs | 11 +-- .../ext/memories/src/tools/ad_hoc_note.rs | 14 +++- codex-rs/ext/memories/src/tools/list.rs | 17 ++++- codex-rs/ext/memories/src/tools/mod.rs | 14 +++- codex-rs/ext/memories/src/tools/read.rs | 20 +++++- codex-rs/ext/memories/src/tools/search.rs | 19 +++-- 9 files changed, 154 insertions(+), 22 deletions(-) create mode 100644 codex-rs/ext/memories/src/metrics.rs diff --git a/codex-rs/ext/memories/src/extension.rs b/codex-rs/ext/memories/src/extension.rs index 67d87fd125..3f0e7e89c6 100644 --- a/codex-rs/ext/memories/src/extension.rs +++ b/codex-rs/ext/memories/src/extension.rs @@ -20,14 +20,12 @@ use crate::tools; /// Contributes Codex memory read-path prompt context and memory read tools. #[derive(Clone, Default)] pub(crate) struct MemoriesExtension { - _metrics_client: Option, + metrics_client: Option, } impl MemoriesExtension { fn new(metrics_client: Option) -> Self { - Self { - _metrics_client: metrics_client, - } + Self { metrics_client } } } @@ -103,7 +101,10 @@ impl ToolContributor for MemoriesExtension { return Vec::new(); } - tools::memory_tools(LocalMemoriesBackend::from_codex_home(&config.codex_home)) + tools::memory_tools( + LocalMemoriesBackend::from_codex_home(&config.codex_home), + self.metrics_client.clone(), + ) } } diff --git a/codex-rs/ext/memories/src/lib.rs b/codex-rs/ext/memories/src/lib.rs index 50dcfb090c..d17c4c4956 100644 --- a/codex-rs/ext/memories/src/lib.rs +++ b/codex-rs/ext/memories/src/lib.rs @@ -1,6 +1,7 @@ mod backend; mod extension; mod local; +mod metrics; mod prompts; mod schema; mod tools; diff --git a/codex-rs/ext/memories/src/metrics.rs b/codex-rs/ext/memories/src/metrics.rs new file mode 100644 index 0000000000..610a3cb9ed --- /dev/null +++ b/codex-rs/ext/memories/src/metrics.rs @@ -0,0 +1,69 @@ +use codex_otel::MetricsClient; + +use crate::MEMORY_TOOLS_NAMESPACE; + +pub(crate) const MEMORIES_TOOL_CALL_METRIC: &str = "codex.memories.tool.call"; + +pub(crate) fn record_tool_call( + metrics_client: Option<&MetricsClient>, + operation: &str, + scope: &str, + success: bool, + truncated: &str, +) { + let Some(metrics_client) = metrics_client else { + return; + }; + + let tool = format!("{MEMORY_TOOLS_NAMESPACE}{operation}"); + let _ = metrics_client.counter( + MEMORIES_TOOL_CALL_METRIC, + /*inc*/ 1, + &[ + ("tool", tool.as_str()), + ("operation", operation), + ("scope", scope), + ("status", status_tag(success)), + ("truncated", truncated), + ], + ); +} + +pub(crate) fn scope_from_path(path: &str) -> &'static str { + let path = path.trim_matches('/'); + let path = path.strip_prefix("./").unwrap_or(path); + + if path.is_empty() { + "root" + } else if path == "MEMORY.md" { + "memory_md" + } else if path == "memory_summary.md" { + "memory_summary" + } else if path == "raw_memories.md" { + "raw_memories" + } else if path == "rollout_summaries" || path.starts_with("rollout_summaries/") { + "rollout_summaries" + } else if path == "skills" || path.starts_with("skills/") { + "skills" + } else if path == "extensions/ad_hoc/notes" || path.starts_with("extensions/ad_hoc/notes/") { + "ad_hoc_notes" + } else { + "other" + } +} + +pub(crate) fn scope_from_optional_path(path: Option<&str>, default: &'static str) -> &'static str { + path.map_or(default, scope_from_path) +} + +pub(crate) fn truncated_tag(truncated: Option) -> &'static str { + match truncated { + Some(true) => "true", + Some(false) => "false", + None => "unknown", + } +} + +fn status_tag(success: bool) -> &'static str { + if success { "succeeded" } else { "failed" } +} diff --git a/codex-rs/ext/memories/src/tests.rs b/codex-rs/ext/memories/src/tests.rs index 0c3380d273..97f54f45ae 100644 --- a/codex-rs/ext/memories/src/tests.rs +++ b/codex-rs/ext/memories/src/tests.rs @@ -415,10 +415,13 @@ async fn search_tool_rejects_legacy_single_query() { fn memory_tool(memory_root: &Path, tool_name: &str) -> Arc> { let expected_tool_name = memory_tool_name(tool_name); - crate::tools::memory_tools(LocalMemoriesBackend::from_memory_root(memory_root)) - .into_iter() - .find(|tool| tool.tool_name() == expected_tool_name) - .unwrap_or_else(|| panic!("{tool_name} tool should be registered")) + crate::tools::memory_tools( + LocalMemoriesBackend::from_memory_root(memory_root), + /*metrics_client*/ None, + ) + .into_iter() + .find(|tool| tool.tool_name() == expected_tool_name) + .unwrap_or_else(|| panic!("{tool_name} tool should be registered")) } fn memory_tool_name(tool_name: &str) -> ToolName { diff --git a/codex-rs/ext/memories/src/tools/ad_hoc_note.rs b/codex-rs/ext/memories/src/tools/ad_hoc_note.rs index 428727dbbd..a6712a4022 100644 --- a/codex-rs/ext/memories/src/tools/ad_hoc_note.rs +++ b/codex-rs/ext/memories/src/tools/ad_hoc_note.rs @@ -3,6 +3,7 @@ use codex_extension_api::ToolCall; use codex_extension_api::ToolExecutor; use codex_extension_api::ToolName; use codex_extension_api::ToolSpec; +use codex_otel::MetricsClient; use schemars::JsonSchema; use serde::Deserialize; use serde_json::json; @@ -11,6 +12,7 @@ use crate::ADD_AD_HOC_NOTE_TOOL_NAME; use crate::backend::AddAdHocMemoryNoteRequest; use crate::backend::AddAdHocMemoryNoteResponse; use crate::backend::MemoriesBackend; +use crate::metrics::record_tool_call; use super::backend_error_to_function_call; use super::memory_function_tool; @@ -36,6 +38,7 @@ struct AddAdHocNoteArgs { #[derive(Clone)] pub(super) struct AddAdHocNoteTool { pub(super) backend: B, + pub(super) metrics_client: Option, } #[async_trait::async_trait] @@ -66,8 +69,15 @@ where filename: args.filename, note: args.note, }) - .await - .map_err(backend_error_to_function_call)?; + .await; + record_tool_call( + self.metrics_client.as_ref(), + ADD_AD_HOC_NOTE_TOOL_NAME, + "ad_hoc_notes", + response.is_ok(), + "not_applicable", + ); + let response = response.map_err(backend_error_to_function_call)?; Ok(Box::new(JsonToolOutput::new(json!(response)))) } } diff --git a/codex-rs/ext/memories/src/tools/list.rs b/codex-rs/ext/memories/src/tools/list.rs index 5133790763..301c7cab71 100644 --- a/codex-rs/ext/memories/src/tools/list.rs +++ b/codex-rs/ext/memories/src/tools/list.rs @@ -3,6 +3,7 @@ use codex_extension_api::ToolCall; use codex_extension_api::ToolExecutor; use codex_extension_api::ToolName; use codex_extension_api::ToolSpec; +use codex_otel::MetricsClient; use schemars::JsonSchema; use serde::Deserialize; use serde_json::json; @@ -13,6 +14,9 @@ use crate::MAX_LIST_RESULTS; use crate::backend::ListMemoriesRequest; use crate::backend::ListMemoriesResponse; use crate::backend::MemoriesBackend; +use crate::metrics::record_tool_call; +use crate::metrics::scope_from_optional_path; +use crate::metrics::truncated_tag; use super::backend_error_to_function_call; use super::clamp_max_results; @@ -32,6 +36,7 @@ struct ListArgs { #[derive(Clone)] pub(super) struct ListTool { pub(super) backend: B, + pub(super) metrics_client: Option, } #[async_trait::async_trait] @@ -57,6 +62,7 @@ where { let backend = self.backend.clone(); let args: ListArgs = parse_args(&call)?; + let scope = scope_from_optional_path(args.path.as_deref(), "root"); let response = backend .list(ListMemoriesRequest { path: args.path, @@ -67,8 +73,15 @@ where MAX_LIST_RESULTS, ), }) - .await - .map_err(backend_error_to_function_call)?; + .await; + record_tool_call( + self.metrics_client.as_ref(), + LIST_TOOL_NAME, + scope, + response.is_ok(), + truncated_tag(response.as_ref().ok().map(|response| response.truncated)), + ); + let response = response.map_err(backend_error_to_function_call)?; Ok(Box::new(JsonToolOutput::new(json!(response)))) } } diff --git a/codex-rs/ext/memories/src/tools/mod.rs b/codex-rs/ext/memories/src/tools/mod.rs index a23bce76e1..768f278866 100644 --- a/codex-rs/ext/memories/src/tools/mod.rs +++ b/codex-rs/ext/memories/src/tools/mod.rs @@ -7,6 +7,7 @@ use codex_extension_api::ToolExecutor; use codex_extension_api::ToolName; use codex_extension_api::ToolSpec; use codex_extension_api::parse_tool_input_schema; +use codex_otel::MetricsClient; use codex_tools::ResponsesApiNamespace; use codex_tools::ResponsesApiNamespaceTool; use codex_tools::default_namespace_description; @@ -24,21 +25,30 @@ mod list; mod read; mod search; -pub(crate) fn memory_tools(backend: B) -> Vec>> +pub(crate) fn memory_tools( + backend: B, + metrics_client: Option, +) -> Vec>> where B: MemoriesBackend, { vec![ Arc::new(ad_hoc_note::AddAdHocNoteTool { backend: backend.clone(), + metrics_client: metrics_client.clone(), }), Arc::new(list::ListTool { backend: backend.clone(), + metrics_client: metrics_client.clone(), }), Arc::new(read::ReadTool { backend: backend.clone(), + metrics_client: metrics_client.clone(), + }), + Arc::new(search::SearchTool { + backend, + metrics_client, }), - Arc::new(search::SearchTool { backend }), ] } diff --git a/codex-rs/ext/memories/src/tools/read.rs b/codex-rs/ext/memories/src/tools/read.rs index f32e1ba82d..33ede3c600 100644 --- a/codex-rs/ext/memories/src/tools/read.rs +++ b/codex-rs/ext/memories/src/tools/read.rs @@ -3,6 +3,7 @@ use codex_extension_api::ToolCall; use codex_extension_api::ToolExecutor; use codex_extension_api::ToolName; use codex_extension_api::ToolSpec; +use codex_otel::MetricsClient; use schemars::JsonSchema; use serde::Deserialize; use serde_json::json; @@ -12,6 +13,9 @@ use crate::READ_TOOL_NAME; use crate::backend::MemoriesBackend; use crate::backend::ReadMemoryRequest; use crate::backend::ReadMemoryResponse; +use crate::metrics::record_tool_call; +use crate::metrics::scope_from_path; +use crate::metrics::truncated_tag; use super::backend_error_to_function_call; use super::memory_function_tool; @@ -31,6 +35,7 @@ struct ReadArgs { #[derive(Clone)] pub(super) struct ReadTool { pub(super) backend: B, + pub(super) metrics_client: Option, } #[async_trait::async_trait] @@ -56,15 +61,24 @@ where { let backend = self.backend.clone(); let args: ReadArgs = parse_args(&call)?; + let path = args.path; + let scope = scope_from_path(path.as_str()); let response = backend .read(ReadMemoryRequest { - path: args.path, + path: path.clone(), line_offset: args.line_offset.unwrap_or(1), max_lines: args.max_lines, max_tokens: DEFAULT_READ_MAX_TOKENS, }) - .await - .map_err(backend_error_to_function_call)?; + .await; + record_tool_call( + self.metrics_client.as_ref(), + READ_TOOL_NAME, + scope, + response.is_ok(), + truncated_tag(response.as_ref().ok().map(|response| response.truncated)), + ); + let response = response.map_err(backend_error_to_function_call)?; Ok(Box::new(JsonToolOutput::new(json!(response)))) } } diff --git a/codex-rs/ext/memories/src/tools/search.rs b/codex-rs/ext/memories/src/tools/search.rs index b94f937d95..1d05072703 100644 --- a/codex-rs/ext/memories/src/tools/search.rs +++ b/codex-rs/ext/memories/src/tools/search.rs @@ -3,6 +3,7 @@ use codex_extension_api::ToolCall; use codex_extension_api::ToolExecutor; use codex_extension_api::ToolName; use codex_extension_api::ToolSpec; +use codex_otel::MetricsClient; use schemars::JsonSchema; use serde::Deserialize; use serde_json::json; @@ -14,6 +15,9 @@ use crate::backend::MemoriesBackend; use crate::backend::SearchMatchMode; use crate::backend::SearchMemoriesRequest; use crate::backend::SearchMemoriesResponse; +use crate::metrics::record_tool_call; +use crate::metrics::scope_from_optional_path; +use crate::metrics::truncated_tag; use super::backend_error_to_function_call; use super::clamp_max_results; @@ -40,6 +44,7 @@ struct SearchArgs { #[derive(Clone)] pub(super) struct SearchTool { pub(super) backend: B, + pub(super) metrics_client: Option, } #[async_trait::async_trait] @@ -65,10 +70,16 @@ where { let backend = self.backend.clone(); let args: SearchArgs = parse_args(&call)?; - let response = backend - .search(args.into_request()) - .await - .map_err(backend_error_to_function_call)?; + let scope = scope_from_optional_path(args.path.as_deref(), "all"); + let response = backend.search(args.into_request()).await; + record_tool_call( + self.metrics_client.as_ref(), + SEARCH_TOOL_NAME, + scope, + response.is_ok(), + truncated_tag(response.as_ref().ok().map(|response| response.truncated)), + ); + let response = response.map_err(backend_error_to_function_call)?; Ok(Box::new(JsonToolOutput::new(json!(response)))) } }