clone/codex
1
0
Fork 0
mirror of https://github.com/openai/codex.git synced 2026-05-02 10:26:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: split memories part 2 (#19860)

Browse Source 
Keep extracting memories out of core and moving the write trigger in the
app-server
This is temporary and it should move at the client level as a follow-up
This makes core fully independant from `codex-memories-write`

---------

Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
jif-oai
2026-04-28 13:03:28 +02:00
committed by GitHub
parent fd36838cf3
commit 431ebeaef7
47 changed files with 1932 additions and 2735 deletions
Download Patch File Download Diff File Expand all files Collapse all files

402
codex-rs/core/tests/suite/memories.rs
View File

@@ -1,402 +0,0 @@
use anyhow::Result;
use chrono::Duration as ChronoDuration;
use chrono::Utc;
use codex_features::Feature;
use codex_git_utils::diff_since_latest_init;
use codex_git_utils::reset_git_repository;
use codex_protocol::ThreadId;
use codex_protocol::protocol::EventMsg;
use codex_protocol::protocol::Op;
use codex_protocol::protocol::SessionSource;
use core_test_support::responses::ResponseMock;
use core_test_support::responses::ResponsesRequest;
use core_test_support::responses::ev_assistant_message;
use core_test_support::responses::ev_completed;
use core_test_support::responses::ev_response_created;
use core_test_support::responses::mount_sse_once;
use core_test_support::responses::sse;
use core_test_support::responses::start_mock_server;
use core_test_support::test_codex::TestCodex;
use core_test_support::test_codex::test_codex;
use core_test_support::wait_for_event;
use pretty_assertions::assert_eq;
use std::path::Path;
use std::sync::Arc;
use tempfile::TempDir;
use tokio::time::Duration;
use tokio::time::Instant;
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn memories_startup_phase2_tracks_workspace_diff_across_runs() -> Result<()> {
let server = start_mock_server().await;
let home = Arc::new(TempDir::new()?);
let db = init_state_db(&home).await?;
let memory_root = home.path().join("memories");
let now = Utc::now();
let _thread_a = seed_stage1_output(
db.as_ref(),
home.path(),
now - ChronoDuration::hours(2),
"raw memory A",
"rollout summary A",
"rollout-a",
)
.await?;
let rollout_summaries_root = memory_root.join("rollout_summaries");
tokio::fs::create_dir_all(&rollout_summaries_root).await?;
tokio::fs::write(
memory_root.join("raw_memories.md"),
"# Raw Memories\n\nraw memory A\n",
)
.await?;
tokio::fs::write(
rollout_summaries_root.join("rollout-a.md"),
"git_branch: branch-rollout-a\n\nrollout summary A\n",
)
.await?;
reset_git_repository(&memory_root).await?;
let _thread_b = seed_stage1_output(
db.as_ref(),
home.path(),
now - ChronoDuration::hours(1),
"raw memory B",
"rollout summary B",
"rollout-b",
)
.await?;
let phase2 = mount_sse_once(
&server,
sse(vec![
ev_response_created("resp-phase2"),
ev_assistant_message("msg-phase2", "phase2 complete"),
ev_completed("resp-phase2"),
]),
)
.await;
let codex = build_test_codex(&server, home.clone()).await?;
let request = wait_for_single_request(&phase2).await;
let prompt = phase2_prompt_text(&request);
assert!(
prompt.contains("phase2_workspace_diff.md"),
"expected workspace diff file in prompt: {prompt}"
);
wait_for_phase2_workspace_reset(&memory_root).await?;
let raw_memories = tokio::fs::read_to_string(memory_root.join("raw_memories.md")).await?;
assert!(raw_memories.contains("raw memory B"));
assert!(!raw_memories.contains("raw memory A"));
let rollout_summaries = read_rollout_summary_bodies(&memory_root).await?;
assert_eq!(rollout_summaries.len(), 1);
assert!(
rollout_summaries
.iter()
.any(|summary| summary.contains("rollout summary B"))
);
assert!(
rollout_summaries
.iter()
.any(|summary| summary.contains("git_branch: branch-rollout-b"))
);
assert!(
rollout_summaries
.iter()
.all(|summary| !summary.contains("rollout summary A"))
);
shutdown_test_codex(&codex).await?;
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn memories_startup_phase2_prunes_old_extension_resources() -> Result<()> {
let server = start_mock_server().await;
let home = Arc::new(TempDir::new()?);
let db = init_state_db(&home).await?;
let now = Utc::now();
let _thread_id = seed_stage1_output(
db.as_ref(),
home.path(),
now - ChronoDuration::hours(1),
"raw memory",
"rollout summary",
"rollout",
)
.await?;
let chronicle_resources = home.path().join("memories/extensions/chronicle/resources");
tokio::fs::create_dir_all(&chronicle_resources).await?;
tokio::fs::write(
home.path()
.join("memories/extensions/chronicle/instructions.md"),
"instructions",
)
.await?;
let old_file_name = format!(
"{}-abcd-10min-old.md",
(now - ChronoDuration::days(8)).format("%Y-%m-%dT%H-%M-%S")
);
let old_file = chronicle_resources.join(&old_file_name);
tokio::fs::write(&old_file, "old resource").await?;
let recent_file = chronicle_resources.join(format!(
"{}-abcd-10min-recent.md",
(now - ChronoDuration::days(6)).format("%Y-%m-%dT%H-%M-%S")
));
tokio::fs::write(&recent_file, "recent resource").await?;
let phase2 = mount_sse_once(
&server,
sse(vec![
ev_response_created("resp-phase2"),
ev_assistant_message("msg-phase2", "phase2 complete"),
ev_completed("resp-phase2"),
]),
)
.await;
let codex = build_test_codex(&server, home.clone()).await?;
let request = wait_for_single_request(&phase2).await;
let prompt = phase2_prompt_text(&request);
assert!(
prompt.contains("phase2_workspace_diff.md"),
"expected workspace diff file in prompt: {prompt}"
);
wait_for_phase2_workspace_reset(&home.path().join("memories")).await?;
wait_for_file_removed(&old_file).await?;
assert!(
!tokio::fs::try_exists(&old_file).await?,
"old extension resource should be pruned"
);
assert!(
tokio::fs::try_exists(&recent_file).await?,
"recent extension resource should be retained"
);
shutdown_test_codex(&codex).await?;
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn memories_startup_phase2_prunes_old_extension_resources_without_stage1_input() -> Result<()>
{
let server = start_mock_server().await;
let home = Arc::new(TempDir::new()?);
let db = init_state_db(&home).await?;
db.enqueue_global_consolidation(/*input_watermark*/ 1)
.await?;
let now = Utc::now();
let chronicle_resources = home.path().join("memories/extensions/chronicle/resources");
tokio::fs::create_dir_all(&chronicle_resources).await?;
tokio::fs::write(
home.path()
.join("memories/extensions/chronicle/instructions.md"),
"instructions",
)
.await?;
let old_file_name = format!(
"{}-abcd-10min-old.md",
(now - ChronoDuration::days(8)).format("%Y-%m-%dT%H-%M-%S")
);
let old_file = chronicle_resources.join(&old_file_name);
tokio::fs::write(&old_file, "old resource").await?;
let phase2 = mount_sse_once(
&server,
sse(vec![
ev_response_created("resp-phase2-empty"),
ev_assistant_message("msg-phase2-empty", "phase2 complete"),
ev_completed("resp-phase2-empty"),
]),
)
.await;
let codex = build_test_codex(&server, home.clone()).await?;
let request = wait_for_single_request(&phase2).await;
let prompt = phase2_prompt_text(&request);
assert!(
prompt.contains("phase2_workspace_diff.md"),
"expected workspace diff file in prompt: {prompt}"
);
wait_for_file_removed(&old_file).await?;
wait_for_phase2_workspace_reset(&home.path().join("memories")).await?;
shutdown_test_codex(&codex).await?;
Ok(())
}
async fn build_test_codex(server: &wiremock::MockServer, home: Arc<TempDir>) -> Result<TestCodex> {
#[allow(clippy::expect_used)]
let mut builder = test_codex().with_home(home).with_config(|config| {
config
.features
.enable(Feature::Sqlite)
.expect("test config should allow feature update");
config
.features
.enable(Feature::MemoryTool)
.expect("test config should allow feature update");
config.memories.max_raw_memories_for_consolidation = 1;
});
builder.build(server).await
}
async fn init_state_db(home: &Arc<TempDir>) -> Result<Arc<codex_state::StateRuntime>> {
let db =
codex_state::StateRuntime::init(home.path().to_path_buf(), "test-provider".into()).await?;
db.mark_backfill_complete(/*last_watermark*/ None).await?;
Ok(db)
}
async fn seed_stage1_output(
db: &codex_state::StateRuntime,
codex_home: &Path,
updated_at: chrono::DateTime<Utc>,
raw_memory: &str,
rollout_summary: &str,
rollout_slug: &str,
) -> Result<ThreadId> {
let thread_id = ThreadId::new();
let mut metadata_builder = codex_state::ThreadMetadataBuilder::new(
thread_id,
codex_home.join(format!("rollout-{thread_id}.jsonl")),
updated_at,
SessionSource::Cli,
);
metadata_builder.cwd = codex_home.join(format!("workspace-{rollout_slug}"));
metadata_builder.model_provider = Some("test-provider".to_string());
metadata_builder.git_branch = Some(format!("branch-{rollout_slug}"));
let metadata = metadata_builder.build("test-provider");
db.upsert_thread(&metadata).await?;
seed_stage1_output_for_existing_thread(
db,
thread_id,
updated_at.timestamp(),
raw_memory,
rollout_summary,
Some(rollout_slug),
)
.await?;
Ok(thread_id)
}
async fn wait_for_single_request(mock: &ResponseMock) -> ResponsesRequest {
wait_for_request(mock, /*expected_count*/ 1).await.remove(0)
}
async fn wait_for_file_removed(path: &Path) -> Result<()> {
let deadline = Instant::now() + Duration::from_secs(10);
loop {
if !tokio::fs::try_exists(path).await? {
return Ok(());
}
assert!(
Instant::now() < deadline,
"timed out waiting for {} to be removed",
path.display()
);
tokio::time::sleep(Duration::from_millis(50)).await;
}
}
async fn wait_for_request(mock: &ResponseMock, expected_count: usize) -> Vec<ResponsesRequest> {
let deadline = Instant::now() + Duration::from_secs(10);
loop {
let requests = mock.requests();
if requests.len() >= expected_count {
return requests;
}
assert!(
Instant::now() < deadline,
"timed out waiting for {expected_count} phase2 requests"
);
tokio::time::sleep(Duration::from_millis(50)).await;
}
}
#[allow(clippy::expect_used)]
fn phase2_prompt_text(request: &ResponsesRequest) -> String {
request
.message_input_texts("user")
.into_iter()
.find(|text| text.contains("Memory workspace diff:"))
.expect("phase2 prompt text")
}
async fn wait_for_phase2_workspace_reset(memory_root: &Path) -> Result<()> {
wait_for_file_removed(&memory_root.join("phase2_workspace_diff.md")).await?;
let deadline = Instant::now() + Duration::from_secs(10);
loop {
if let Ok(diff) = diff_since_latest_init(memory_root).await
&& !diff.has_changes()
{
return Ok(());
}
assert!(
Instant::now() < deadline,
"timed out waiting for clean memory workspace baseline"
);
tokio::time::sleep(Duration::from_millis(50)).await;
}
}
async fn seed_stage1_output_for_existing_thread(
db: &codex_state::StateRuntime,
thread_id: ThreadId,
updated_at: i64,
raw_memory: &str,
rollout_summary: &str,
rollout_slug: Option<&str>,
) -> Result<()> {
let owner = ThreadId::new();
let claim = db
.try_claim_stage1_job(
thread_id, owner, updated_at, /*lease_seconds*/ 3_600,
/*max_running_jobs*/ 64,
)
.await?;
let ownership_token = match claim {
codex_state::Stage1JobClaimOutcome::Claimed { ownership_token } => ownership_token,
other => panic!("unexpected stage-1 claim outcome: {other:?}"),
};
assert!(
db.mark_stage1_job_succeeded(
thread_id,
&ownership_token,
updated_at,
raw_memory,
rollout_summary,
rollout_slug,
)
.await?,
"stage-1 success should enqueue global consolidation"
);
Ok(())
}
async fn read_rollout_summary_bodies(memory_root: &Path) -> Result<Vec<String>> {
let mut dir = tokio::fs::read_dir(memory_root.join("rollout_summaries")).await?;
let mut summaries = Vec::new();
while let Some(entry) = dir.next_entry().await? {
summaries.push(tokio::fs::read_to_string(entry.path()).await?);
}
summaries.sort();
Ok(summaries)
}
async fn shutdown_test_codex(test: &TestCodex) -> Result<()> {
test.codex.submit(Op::Shutdown {}).await?;
wait_for_event(&test.codex, |ev| matches!(ev, EventMsg::ShutdownComplete)).await;
Ok(())
}

1
codex-rs/core/tests/suite/mod.rs
View File

@@ -54,7 +54,6 @@ mod items;
mod json_result;
mod live_cli;
mod live_reload;
mod memories;
mod model_overrides;
mod model_switching;
mod model_visible_layout;