clone/codex
1
0
Fork 0
mirror of https://github.com/openai/codex.git synced 2026-05-29 15:30:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

mvoes

Browse Source
This commit is contained in:
jif-oai
2026-04-29 17:17:19 +01:00
parent 63402cbc3b
commit 03c5bfdd28
12 changed files with 985 additions and 388 deletions
Download Patch File Download Diff File Expand all files Collapse all files

219
codex-rs/core/src/context_manager/updates.rs
View File

@@ -13,23 +13,36 @@ use crate::shell::Shell;
use codex_execpolicy::Policy;
use codex_features::Feature;
use codex_journal::Journal;
use codex_journal::JournalContextItem;
use codex_journal::JournalContextKey;
use codex_journal::JournalEntry;
use codex_journal::JournalItem;
use codex_journal::KeyFilter;
use codex_journal::MetadataEntryBuilder;
use codex_journal::PromptMessage;
use codex_journal::PromptMessageRole;
use codex_journal::PromptRenderer;
use codex_protocol::config_types::Personality;
use codex_protocol::models::ContentItem;
use codex_protocol::models::ResponseItem;
use codex_protocol::openai_models::ModelInfo;
use codex_protocol::protocol::TurnContextItem;
const PROMPT_BUNDLE_KEY_PREFIX: &str = "prompt";
const DEVELOPER_BUNDLE: &str = "developer";
const USAGE_HINT_BUNDLE: &str = "usage_hint";
const CONTEXTUAL_USER_BUNDLE: &str = "contextual_user";
const GUARDIAN_BUNDLE: &str = "guardian";
pub(crate) const DEVELOPER_BUNDLE: &str = "developer";
pub(crate) const USAGE_HINT_BUNDLE: &str = "usage_hint";
pub(crate) const CONTEXTUAL_USER_BUNDLE: &str = "contextual_user";
pub(crate) const GUARDIAN_BUNDLE: &str = "guardian";
pub(crate) fn context_prompt_renderer() -> PromptRenderer {
PromptRenderer::new()
.group(KeyFilter::prefix([
PROMPT_BUNDLE_KEY_PREFIX,
DEVELOPER_BUNDLE,
]))
.group(KeyFilter::prefix([
PROMPT_BUNDLE_KEY_PREFIX,
USAGE_HINT_BUNDLE,
]))
.group(KeyFilter::prefix([
PROMPT_BUNDLE_KEY_PREFIX,
CONTEXTUAL_USER_BUNDLE,
]))
}
fn build_environment_update_item(
previous: Option<&TurnContextItem>,
@@ -186,133 +199,19 @@ pub(crate) fn build_model_instructions_update_item(
Some(ModelSwitchInstructions::new(model_instructions).render())
}
pub(crate) fn developer_context_entry(
name: &str,
prompt_order: i64,
text: String,
) -> Option<JournalEntry> {
context_entry(
DEVELOPER_BUNDLE,
name,
prompt_order,
PromptMessage::developer_text(text),
)
}
pub(crate) fn usage_hint_context_entry(
name: &str,
prompt_order: i64,
text: String,
) -> Option<JournalEntry> {
context_entry(
USAGE_HINT_BUNDLE,
name,
prompt_order,
PromptMessage::developer_text(text),
)
}
pub(crate) fn contextual_user_context_entry(
name: &str,
prompt_order: i64,
text: String,
) -> Option<JournalEntry> {
context_entry(
CONTEXTUAL_USER_BUNDLE,
name,
prompt_order,
PromptMessage::user_text(text),
)
}
pub(crate) fn guardian_context_entry(
name: &str,
prompt_order: i64,
text: String,
) -> Option<JournalEntry> {
context_entry(
GUARDIAN_BUNDLE,
name,
prompt_order,
PromptMessage::developer_text(text),
)
}
fn context_entry(
pub(crate) fn context_entry(
bundle: &str,
name: &str,
prompt_order: i64,
message: PromptMessage,
) -> Option<JournalEntry> {
if message.content.is_empty()
|| message.content.iter().all(|item| match item {
ContentItem::InputText { text } | ContentItem::OutputText { text } => {
text.trim().is_empty()
}
ContentItem::InputImage { .. } => false,
})
{
return None;
}
Some(JournalEntry::new(
[PROMPT_BUNDLE_KEY_PREFIX, bundle, name],
JournalContextItem::new(JournalContextKey::new(bundle, name, None), message)
.with_prompt_order(prompt_order),
))
context_entry_builder(bundle, name, message)
.prompt_order(prompt_order)
.build()
}
pub(crate) fn render_context_entries(entries: Vec<JournalEntry>) -> Vec<ResponseItem> {
let flattened = match Journal::from_entries(entries).flatten() {
Ok(flattened) => flattened,
Err(error) => unreachable!("context-only journal entries should flatten: {error}"),
};
let mut rendered = Vec::new();
let mut current_bundle: Option<Vec<String>> = None;
let mut current_role: Option<PromptMessageRole> = None;
let mut current_content = Vec::new();
for entry in flattened.entries() {
let JournalItem::Context(item) = &entry.item else {
continue;
};
let bundle = entry
.key
.parts()
.iter()
.take(2)
.cloned()
.collect::<Vec<_>>();
if current_bundle.as_ref() != Some(&bundle) || current_role != Some(item.message.role) {
flush_prompt_message(&mut rendered, &mut current_role, &mut current_content);
current_bundle = Some(bundle);
current_role = Some(item.message.role);
}
current_content.extend(item.message.content.clone());
}
flush_prompt_message(&mut rendered, &mut current_role, &mut current_content);
rendered
}
fn flush_prompt_message(
rendered: &mut Vec<ResponseItem>,
role: &mut Option<PromptMessageRole>,
content: &mut Vec<codex_protocol::models::ContentItem>,
) {
let Some(role) = role.take() else {
return;
};
if content.is_empty() {
return;
}
rendered.push(ResponseItem::from(PromptMessage::new(
role,
std::mem::take(content),
)));
fn context_entry_builder(bundle: &str, name: &str, message: PromptMessage) -> MetadataEntryBuilder {
Journal::metadata_entry_builder([PROMPT_BUNDLE_KEY_PREFIX, bundle, name], message)
}
pub(crate) fn build_settings_update_entries(
@@ -333,34 +232,70 @@ pub(crate) fn build_settings_update_entries(
build_model_instructions_update_item(previous_turn_settings, next).and_then(|text| {
// Keep model-switch instructions first so model-specific guidance is read before
// any other context diffs on this turn.
developer_context_entry("model_switch", 10, text)
context_entry(
DEVELOPER_BUNDLE,
"model_switch",
10,
PromptMessage::developer_text(text),
)
})
{
entries.push(item);
}
if let Some(item) = build_permissions_update_item(previous, next, exec_policy)
.and_then(|text| developer_context_entry("permissions", 20, text))
if let Some(item) =
build_permissions_update_item(previous, next, exec_policy).and_then(|text| {
context_entry(
DEVELOPER_BUNDLE,
"permissions",
20,
PromptMessage::developer_text(text),
)
})
{
entries.push(item);
}
if let Some(item) = build_collaboration_mode_update_item(previous, next)
.and_then(|text| developer_context_entry("collaboration_mode", 30, text))
{
if let Some(item) = build_collaboration_mode_update_item(previous, next).and_then(|text| {
context_entry(
DEVELOPER_BUNDLE,
"collaboration_mode",
30,
PromptMessage::developer_text(text),
)
}) {
entries.push(item);
}
if let Some(item) = build_realtime_update_item(previous, previous_turn_settings, next)
.and_then(|text| developer_context_entry("realtime", 40, text))
if let Some(item) =
build_realtime_update_item(previous, previous_turn_settings, next).and_then(|text| {
context_entry(
DEVELOPER_BUNDLE,
"realtime",
40,
PromptMessage::developer_text(text),
)
})
{
entries.push(item);
}
if let Some(item) = build_personality_update_item(previous, next, personality_feature_enabled)
.and_then(|text| developer_context_entry("personality", 50, text))
.and_then(|text| {
context_entry(
DEVELOPER_BUNDLE,
"personality",
50,
PromptMessage::developer_text(text),
)
})
{
entries.push(item);
}
if let Some(item) = build_environment_update_item(previous, next, shell)
.and_then(|text| contextual_user_context_entry("environment", 60, text))
{
if let Some(item) = build_environment_update_item(previous, next, shell).and_then(|text| {
context_entry(
CONTEXTUAL_USER_BUNDLE,
"environment",
60,
PromptMessage::user_text(text),
)
}) {
entries.push(item);
}

109
codex-rs/core/src/session/mod.rs
View File

@@ -59,6 +59,7 @@ use codex_features::Feature;
use codex_features::unstable_features_warning_event;
use codex_hooks::Hooks;
use codex_hooks::HooksConfig;
use codex_journal::PromptMessage;
use codex_login::AuthManager;
use codex_login::CodexAuth;
use codex_login::auth_env_telemetry::collect_auth_env_telemetry;
@@ -1437,7 +1438,12 @@ impl Session {
exec_policy.as_ref(),
self.features.enabled(Feature::Personality),
);
crate::context_manager::updates::render_context_entries(entries)
let resolved = match codex_journal::Journal::from_entries(entries).resolve() {
Ok(resolved) => resolved,
Err(error) => unreachable!("settings update entries should resolve: {error}"),
};
crate::context_manager::updates::context_prompt_renderer()
.render_metadata(resolved.metadata())
}
/// Persist the event to rollout and send it to clients.
@@ -2503,9 +2509,16 @@ impl Session {
&self,
turn_context: &TurnContext,
) -> Vec<ResponseItem> {
crate::context_manager::updates::render_context_entries(
let resolved = match codex_journal::Journal::from_entries(
self.build_initial_context_entries(turn_context).await,
)
.resolve()
{
Ok(resolved) => resolved,
Err(error) => unreachable!("initial context entries should resolve: {error}"),
};
crate::context_manager::updates::context_prompt_renderer()
.render_metadata(resolved.metadata())
}
#[expect(
@@ -2539,10 +2552,11 @@ impl Session {
previous_turn_settings.as_ref(),
turn_context,
)
&& let Some(entry) = crate::context_manager::updates::developer_context_entry(
&& let Some(entry) = crate::context_manager::updates::context_entry(
crate::context_manager::updates::DEVELOPER_BUNDLE,
"model_switch",
10,
model_switch_message,
PromptMessage::developer_text(model_switch_message),
)
{
entries.push(entry);
@@ -2562,10 +2576,11 @@ impl Session {
.enabled(Feature::RequestPermissionsTool),
)
.render();
if let Some(entry) = crate::context_manager::updates::developer_context_entry(
if let Some(entry) = crate::context_manager::updates::context_entry(
crate::context_manager::updates::DEVELOPER_BUNDLE,
"permissions",
20,
permissions_instructions,
PromptMessage::developer_text(permissions_instructions),
) {
entries.push(entry);
}
@@ -2577,10 +2592,11 @@ impl Session {
if !separate_guardian_developer_message
&& let Some(developer_instructions) = turn_context.developer_instructions.as_deref()
&& !developer_instructions.is_empty()
&& let Some(entry) = crate::context_manager::updates::developer_context_entry(
&& let Some(entry) = crate::context_manager::updates::context_entry(
crate::context_manager::updates::DEVELOPER_BUNDLE,
"developer_instructions",
30,
developer_instructions.to_string(),
PromptMessage::developer_text(developer_instructions.to_string()),
)
{
entries.push(entry);
@@ -2590,10 +2606,11 @@ impl Session {
&& turn_context.config.memories.use_memories
&& let Some(memory_prompt) =
build_memory_tool_developer_instructions(&turn_context.config.codex_home).await
&& let Some(entry) = crate::context_manager::updates::developer_context_entry(
&& let Some(entry) = crate::context_manager::updates::context_entry(
crate::context_manager::updates::DEVELOPER_BUNDLE,
"memory_tool",
40,
memory_prompt,
PromptMessage::developer_text(memory_prompt),
)
{
entries.push(entry);
@@ -2601,10 +2618,11 @@ impl Session {
// Add developer instructions from collaboration_mode if they exist and are non-empty
if let Some(collab_instructions) =
CollaborationModeInstructions::from_collaboration_mode(&collaboration_mode)
&& let Some(entry) = crate::context_manager::updates::developer_context_entry(
&& let Some(entry) = crate::context_manager::updates::context_entry(
crate::context_manager::updates::DEVELOPER_BUNDLE,
"collaboration_mode",
50,
collab_instructions.render(),
PromptMessage::developer_text(collab_instructions.render()),
)
{
entries.push(entry);
@@ -2613,10 +2631,11 @@ impl Session {
reference_context_item.as_ref(),
previous_turn_settings.as_ref(),
turn_context,
) && let Some(entry) = crate::context_manager::updates::developer_context_entry(
) && let Some(entry) = crate::context_manager::updates::context_entry(
crate::context_manager::updates::DEVELOPER_BUNDLE,
"realtime",
60,
realtime_update,
PromptMessage::developer_text(realtime_update),
) {
entries.push(entry);
}
@@ -2632,10 +2651,13 @@ impl Session {
&model_info,
personality,
)
&& let Some(entry) = crate::context_manager::updates::developer_context_entry(
&& let Some(entry) = crate::context_manager::updates::context_entry(
crate::context_manager::updates::DEVELOPER_BUNDLE,
"personality",
70,
PersonalitySpecInstructions::new(personality_message).render(),
PromptMessage::developer_text(
PersonalitySpecInstructions::new(personality_message).render(),
),
)
{
entries.push(entry);
@@ -2651,10 +2673,11 @@ impl Session {
.await;
if let Some(apps_instructions) =
AppsInstructions::from_connectors(&accessible_and_enabled_connectors)
&& let Some(entry) = crate::context_manager::updates::developer_context_entry(
&& let Some(entry) = crate::context_manager::updates::context_entry(
crate::context_manager::updates::DEVELOPER_BUNDLE,
"apps",
80,
apps_instructions.render(),
PromptMessage::developer_text(apps_instructions.render()),
)
{
entries.push(entry);
@@ -2680,10 +2703,11 @@ impl Session {
})
.await;
}
if let Some(entry) = crate::context_manager::updates::developer_context_entry(
if let Some(entry) = crate::context_manager::updates::context_entry(
crate::context_manager::updates::DEVELOPER_BUNDLE,
"skills",
90,
skills_instructions.render(),
PromptMessage::developer_text(skills_instructions.render()),
) {
entries.push(entry);
}
@@ -2696,10 +2720,11 @@ impl Session {
.await;
if let Some(plugin_instructions) =
AvailablePluginsInstructions::from_plugins(loaded_plugins.capability_summaries())
&& let Some(entry) = crate::context_manager::updates::developer_context_entry(
&& let Some(entry) = crate::context_manager::updates::context_entry(
crate::context_manager::updates::DEVELOPER_BUNDLE,
"plugins",
100,
plugin_instructions.render(),
PromptMessage::developer_text(plugin_instructions.render()),
)
{
entries.push(entry);
@@ -2708,23 +2733,27 @@ impl Session {
&& let Some(commit_message_instruction) = commit_message_trailer_instruction(
turn_context.config.commit_attribution.as_deref(),
)
&& let Some(entry) = crate::context_manager::updates::developer_context_entry(
&& let Some(entry) = crate::context_manager::updates::context_entry(
crate::context_manager::updates::DEVELOPER_BUNDLE,
"commit_message_trailer",
110,
commit_message_instruction,
PromptMessage::developer_text(commit_message_instruction),
)
{
entries.push(entry);
}
if let Some(user_instructions) = turn_context.user_instructions.as_deref()
&& let Some(entry) = crate::context_manager::updates::contextual_user_context_entry(
&& let Some(entry) = crate::context_manager::updates::context_entry(
crate::context_manager::updates::CONTEXTUAL_USER_BUNDLE,
"user_instructions",
130,
UserInstructions {
text: user_instructions.to_string(),
directory: turn_context.cwd.to_string_lossy().into_owned(),
}
.render(),
PromptMessage::user_text(
UserInstructions {
text: user_instructions.to_string(),
directory: turn_context.cwd.to_string_lossy().into_owned(),
}
.render(),
),
)
{
entries.push(entry);
@@ -2735,12 +2764,18 @@ impl Session {
.agent_control
.format_environment_context_subagents(self.conversation_id)
.await;
if let Some(entry) = crate::context_manager::updates::contextual_user_context_entry(
if let Some(entry) = crate::context_manager::updates::context_entry(
crate::context_manager::updates::CONTEXTUAL_USER_BUNDLE,
"environment",
140,
crate::context::EnvironmentContext::from_turn_context(turn_context, shell.as_ref())
PromptMessage::user_text(
crate::context::EnvironmentContext::from_turn_context(
turn_context,
shell.as_ref(),
)
.with_subagents(subagents)
.render(),
),
) {
entries.push(entry);
}
@@ -2750,10 +2785,11 @@ impl Session {
multi_agents::usage_hint_text(turn_context, &session_source);
if let Some(usage_hint_text) = multi_agent_v2_usage_hint_text
&& let Some(entry) = crate::context_manager::updates::usage_hint_context_entry(
&& let Some(entry) = crate::context_manager::updates::context_entry(
crate::context_manager::updates::USAGE_HINT_BUNDLE,
"multi_agent_v2",
120,
usage_hint_text.to_string(),
PromptMessage::developer_text(usage_hint_text.to_string()),
)
{
entries.push(entry);
@@ -2763,10 +2799,11 @@ impl Session {
if separate_guardian_developer_message
&& let Some(developer_instructions) = turn_context.developer_instructions.as_deref()
&& !developer_instructions.is_empty()
&& let Some(entry) = crate::context_manager::updates::guardian_context_entry(
&& let Some(entry) = crate::context_manager::updates::context_entry(
crate::context_manager::updates::GUARDIAN_BUNDLE,
"developer_instructions",
150,
developer_instructions.to_string(),
PromptMessage::developer_text(developer_instructions.to_string()),
)
{
entries.push(entry);

8
codex-rs/core/src/state/history.rs
View File

@@ -3,9 +3,9 @@ use crate::event_mapping::is_contextual_dev_message_content;
use crate::event_mapping::is_contextual_user_message_content;
use crate::session::turn_context::TurnContext;
use codex_journal::Journal;
use codex_journal::JournalHistoryItem;
use codex_journal::JournalItem;
use codex_journal::JournalKey;
use codex_journal::JournalTranscriptItem;
use codex_journal::history as thread_history;
use codex_journal::history::estimate_item_token_count;
use codex_journal::history::estimate_response_item_model_visible_bytes;
@@ -52,8 +52,8 @@ pub(crate) fn raw_items(journal: &Journal) -> Vec<ResponseItem> {
.entries()
.iter()
.filter_map(|entry| match &entry.item {
JournalItem::History(item) => Some(item.item.clone()),
JournalItem::Context(_) | JournalItem::Checkpoint(_) => None,
JournalItem::Transcript(item) => Some(item.item.clone()),
JournalItem::Metadata(_) | JournalItem::Checkpoint(_) => None,
})
.collect()
}
@@ -232,7 +232,7 @@ pub(crate) fn get_total_token_usage_breakdown(
}
fn push_history_item(journal: &mut Journal, item: ResponseItem) {
let history_item = JournalHistoryItem::new(item);
let history_item = JournalTranscriptItem::new(item);
let key = JournalKey::new(vec!["history".to_string(), history_item.id.clone()]);
journal.add(key, history_item);
}

1
codex-rs/journal/Cargo.toml
View File

@@ -3,6 +3,7 @@ name = "codex-journal"
version.workspace = true
edition.workspace = true
license.workspace = true
readme = "README.md"
[lints]
workspace = true

86
codex-rs/journal/README.md Normal file
View File

@@ -0,0 +1,86 @@
# codex-journal
`codex-journal` is the typed journal model behind prompt metadata, transcript rewriting, forked views,
and prompt-ready projection in Codex.
## Model
A `Journal` stores one append-only sequence of `JournalEntry` values. Each entry is one of:
- prompt metadata: keyed entries that can replace older entries with the same key
- transcript: ordered prompt-visible items
- checkpoints: transcript rewrite operations such as prefix replacement or truncation
The append-only journal is the source of truth. Derived views are produced by resolving it.
## Resolved Views
`Journal::resolve()` returns a `ResolvedJournal` with two typed views:
- `ResolvedMetadata`: effective prompt-metadata entries after key-based replacement
- `ResolvedTranscript`: effective transcript after checkpoint application
These two views come from the same journal, but they behave differently:
- metadata is deduplicated by key and ordered by `prompt_order`
- transcript preserves order and is rewritten only by checkpoints
## Building Metadata Entries
Use `Journal::metadata_entry_builder(...)` when you want explicit control over metadata:
```rust
use codex_journal::Journal;
use codex_journal::JournalContextAudience;
use codex_journal::PromptMessage;
let entry = Journal::metadata_entry_builder(
["prompt", "developer", "permissions"],
PromptMessage::developer_text("sandbox is workspace-write"),
)
.prompt_order(20)
.audience(JournalContextAudience::All)
.build();
```
`Journal::metadata_entry(...)` remains available as a shorthand for the common case.
## Rendering
Prompt rendering lives in `PromptRenderer`, not in `Journal` itself. Grouping is explicit and
applied only to consecutive resolved metadata entries:
```rust
use codex_journal::Journal;
use codex_journal::KeyFilter;
use codex_journal::PromptMessage;
use codex_journal::PromptRenderer;
let journal = Journal::from_entries(vec![
Journal::metadata_entry(
["prompt", "developer", "one"],
10,
PromptMessage::developer_text("first"),
).unwrap(),
Journal::metadata_entry(
["prompt", "developer", "two"],
20,
PromptMessage::developer_text("second"),
).unwrap(),
]);
let resolved = journal.resolve().unwrap();
let prompt = PromptRenderer::new()
.group(KeyFilter::prefix(["prompt", "developer"]))
.render_metadata(resolved.metadata());
```
## Transformations
`Journal` also supports:
- `filter` and `resolve_with_filter` for key-scoped views
- `flatten` for dropping obsolete entries while keeping the current effective view
- `fork` for producing child views with audience and `on_fork` filtering
- `with_history_window` for keeping only a recent hot history suffix
- `persist_jsonl` / `load_jsonl` for durable JSONL storage

87
codex-rs/journal/src/context_builder.rs Normal file
View File

@@ -0,0 +1,87 @@
use crate::JournalContextAudience;
use crate::JournalContextForkBehavior;
use crate::JournalEntry;
use crate::JournalKey;
use crate::JournalMetadataItem;
use crate::PromptMessage;
use codex_protocol::models::ContentItem;
/// Builder for one prompt-metadata journal entry.
#[derive(Debug, Clone)]
pub struct MetadataEntryBuilder {
key: JournalKey,
message: PromptMessage,
prompt_order: i64,
audience: JournalContextAudience,
on_fork: JournalContextForkBehavior,
tags: Vec<String>,
source: Option<String>,
}
impl MetadataEntryBuilder {
pub(crate) fn new(key: impl Into<JournalKey>, message: impl Into<PromptMessage>) -> Self {
Self {
key: key.into(),
message: message.into(),
prompt_order: 0,
audience: JournalContextAudience::default(),
on_fork: JournalContextForkBehavior::default(),
tags: Vec::new(),
source: None,
}
}
/// Sets the prompt ordering used after resolution.
pub fn prompt_order(mut self, prompt_order: i64) -> Self {
self.prompt_order = prompt_order;
self
}
/// Sets the audience used when projecting or forking the journal.
pub fn audience(mut self, audience: JournalContextAudience) -> Self {
self.audience = audience;
self
}
/// Sets how this context entry behaves when creating a forked journal view.
pub fn on_fork(mut self, on_fork: JournalContextForkBehavior) -> Self {
self.on_fork = on_fork;
self
}
/// Sets arbitrary tags for downstream classification.
pub fn tags(mut self, tags: Vec<String>) -> Self {
self.tags = tags;
self
}
/// Records the origin of this prompt-metadata entry.
pub fn source(mut self, source: impl Into<String>) -> Self {
self.source = Some(source.into());
self
}
/// Builds one prompt-metadata entry if the message is non-empty after trimming text content.
pub fn build(self) -> Option<JournalEntry> {
if self.message.content.is_empty()
|| self.message.content.iter().all(|item| match item {
ContentItem::InputText { text } | ContentItem::OutputText { text } => {
text.trim().is_empty()
}
ContentItem::InputImage { .. } => false,
})
{
return None;
}
let mut item = JournalMetadataItem::new(self.message)
.with_prompt_order(self.prompt_order)
.with_audience(self.audience)
.with_on_fork(self.on_fork)
.with_tags(self.tags);
if let Some(source) = self.source {
item = item.with_source(source);
}
Some(JournalEntry::new(self.key, item))
}
}

352
codex-rs/journal/src/journal.rs
View File

@@ -1,18 +1,18 @@
use crate::JournalEntry;
use crate::JournalError;
use crate::JournalItem;
use crate::JournalKey;
use crate::KeyFilter;
use crate::MetadataEntryBuilder;
use crate::PromptView;
use crate::Result;
use codex_protocol::journal::JournalCheckpointItem;
use codex_protocol::journal::JournalContextAudience;
use codex_protocol::journal::JournalContextForkBehavior;
use codex_protocol::journal::JournalContextItem;
use codex_protocol::journal::JournalContextKey;
use codex_protocol::journal::JournalHistoryCursor;
use codex_protocol::journal::JournalHistoryItem;
use codex_protocol::journal::JournalItem;
use codex_protocol::journal::JournalMetadataItem;
use codex_protocol::journal::JournalReplacePrefixCheckpoint;
use codex_protocol::journal::JournalTranscriptItem;
use codex_protocol::journal::JournalTruncateHistoryCheckpoint;
use codex_protocol::models::ResponseItem;
use indexmap::IndexMap;
@@ -63,6 +63,40 @@ impl Journal {
&self.entries
}
/// Starts building one prompt-metadata entry.
pub fn metadata_entry_builder(
key: impl Into<JournalKey>,
message: impl Into<crate::PromptMessage>,
) -> MetadataEntryBuilder {
MetadataEntryBuilder::new(key, message)
}
/// Builds one prompt-metadata entry if the message is non-empty after trimming text content.
pub fn metadata_entry(
key: impl Into<JournalKey>,
prompt_order: i64,
message: impl Into<crate::PromptMessage>,
) -> Option<JournalEntry> {
Self::metadata_entry_builder(key, message)
.prompt_order(prompt_order)
.build()
}
pub fn context_entry_builder(
key: impl Into<JournalKey>,
message: impl Into<crate::PromptMessage>,
) -> MetadataEntryBuilder {
Self::metadata_entry_builder(key, message)
}
pub fn context_entry(
key: impl Into<JournalKey>,
prompt_order: i64,
message: impl Into<crate::PromptMessage>,
) -> Option<JournalEntry> {
Self::metadata_entry(key, prompt_order, message)
}
/// Returns a journal containing only journal entries whose keys match the filter.
pub fn filter(&self, filter: &KeyFilter) -> Self {
let entries = self
@@ -89,8 +123,8 @@ impl Journal {
self.to_prompt_matching_filter(view, None)
}
/// Renders the current effective journal view into model prompt items after selecting
/// only journal entries whose keys match the filter.
/// Renders the current effective journal view into model prompt items after selecting only
/// journal entries whose keys match the filter.
pub fn to_prompt_with_filter(
&self,
view: &PromptView,
@@ -115,14 +149,8 @@ impl Journal {
/// This is the first building block for a rolling in-memory window: callers can
/// persist the full journal elsewhere, then keep only the flattened journal hot.
pub fn flatten(&self) -> Result<Self> {
let resolved = self.resolve(None)?;
Ok(Self::from_entries(
resolved
.contexts
.into_iter()
.chain(resolved.history)
.collect(),
))
let resolved = self.resolve()?;
Ok(Self::from_entries(resolved.into_entries()))
}
/// Keeps only the current effective journal view plus the history suffix that starts
@@ -131,13 +159,16 @@ impl Journal {
/// This is a lightweight rolling-window helper: callers can persist the full
/// journal on disk, then keep only a recent hot suffix in memory.
pub fn with_history_window(&self, start: &JournalHistoryCursor) -> Result<Self> {
let resolved = self.resolve(None)?;
let start_index = resolve_cursor(resolved.history.as_slice(), start)?;
let resolved = self.resolve()?;
let start_index = resolve_cursor(resolved.transcript().entries(), start)?;
let ResolvedJournal {
metadata,
transcript,
} = resolved;
Ok(Self::from_entries(
resolved
.contexts
metadata
.into_iter()
.chain(resolved.history.into_iter().skip(start_index))
.chain(transcript.into_iter().skip(start_index))
.collect(),
))
}
@@ -175,44 +206,60 @@ impl Journal {
Ok(Self::from_entries(entries))
}
/// Resolves the current effective journal into prompt metadata and transcript views.
pub fn resolve(&self) -> Result<ResolvedJournal> {
self.resolve_filter(None)
}
/// Resolves the current effective journal after selecting only entries whose keys match the
/// filter.
pub fn resolve_with_filter(&self, filter: &KeyFilter) -> Result<ResolvedJournal> {
self.resolve_filter(Some(filter))
}
fn to_prompt_matching_filter(
&self,
view: &PromptView,
filter: Option<&KeyFilter>,
) -> Result<Vec<ResponseItem>> {
let resolved = self.resolve(filter)?;
let resolved = self.resolve_filter(filter)?;
let mut prompt: Vec<ResponseItem> = resolved
.contexts
.metadata
.into_iter()
.filter(|entry| context_visible_in_view(context_item(entry), view))
.filter(|entry| metadata_visible_in_view(metadata_item(entry), view))
.map(|entry| match entry.item {
JournalItem::Context(item) => ResponseItem::from(item),
_ => unreachable!("resolved context entries must be context items"),
JournalItem::Metadata(item) => ResponseItem::from(item),
_ => unreachable!("resolved metadata entries must be metadata items"),
})
.collect();
prompt.extend(resolved.history.into_iter().map(|entry| match entry.item {
JournalItem::History(item) => ResponseItem::from(item),
_ => unreachable!("resolved history entries must be history items"),
}));
prompt.extend(
resolved
.transcript
.into_iter()
.map(|entry| match entry.item {
JournalItem::Transcript(item) => ResponseItem::from(item),
_ => unreachable!("resolved transcript entries must be transcript items"),
}),
);
Ok(prompt)
}
fn fork_matching_filter(&self, view: &PromptView, filter: Option<&KeyFilter>) -> Result<Self> {
let resolved = self.resolve(filter)?;
let resolved = self.resolve_filter(filter)?;
Ok(Self::from_entries(
resolved
.contexts
.metadata
.into_iter()
.filter(|entry| context_item(entry).on_fork == JournalContextForkBehavior::Keep)
.filter(|entry| context_visible_in_view(context_item(entry), view))
.chain(resolved.history)
.filter(|entry| metadata_item(entry).on_fork == JournalContextForkBehavior::Keep)
.filter(|entry| metadata_visible_in_view(metadata_item(entry), view))
.chain(resolved.transcript)
.collect(),
))
}
fn resolve(&self, filter: Option<&KeyFilter>) -> Result<ResolvedJournal> {
let mut history = Vec::<JournalEntry>::new();
let mut latest_context_by_key = IndexMap::<JournalContextKey, (usize, JournalEntry)>::new();
fn resolve_filter(&self, filter: Option<&KeyFilter>) -> Result<ResolvedJournal> {
let mut transcript = Vec::<JournalEntry>::new();
let mut latest_metadata_by_key = IndexMap::<JournalKey, (usize, JournalEntry)>::new();
for (index, entry) in self.entries.iter().enumerate() {
if let Some(filter) = filter
@@ -221,39 +268,192 @@ impl Journal {
continue;
}
match &entry.item {
JournalItem::History(_) => history.push(entry.clone()),
JournalItem::Context(context_item) => {
latest_context_by_key.insert(context_item.key.clone(), (index, entry.clone()));
JournalItem::Transcript(_) => transcript.push(entry.clone()),
JournalItem::Metadata(_) => {
latest_metadata_by_key.insert(entry.key.clone(), (index, entry.clone()));
}
JournalItem::Checkpoint(checkpoint) => {
apply_checkpoint(&mut history, &entry.key, checkpoint)?;
apply_checkpoint(&mut transcript, &entry.key, checkpoint)?;
}
}
}
let mut contexts = latest_context_by_key.into_values().collect::<Vec<_>>();
contexts.sort_by(|(left_index, left_entry), (right_index, right_entry)| {
context_item(left_entry)
let mut metadata = latest_metadata_by_key.into_values().collect::<Vec<_>>();
metadata.sort_by(|(left_index, left_entry), (right_index, right_entry)| {
metadata_item(left_entry)
.prompt_order
.cmp(&context_item(right_entry).prompt_order)
.cmp(&metadata_item(right_entry).prompt_order)
.then_with(|| left_index.cmp(right_index))
});
Ok(ResolvedJournal {
contexts: contexts.into_iter().map(|(_, entry)| entry).collect(),
history,
})
Ok(ResolvedJournal::new(
metadata.into_iter().map(|(_, entry)| entry).collect(),
transcript,
))
}
}
#[derive(Debug)]
struct ResolvedJournal {
contexts: Vec<JournalEntry>,
history: Vec<JournalEntry>,
/// Effective journal view after deduplicating prompt context and applying history checkpoints.
///
/// `contexts` and `history` are derived from the same append-only source of truth, but they have
/// different semantics:
///
/// - [`ResolvedMetadata`] contains keyed prompt-metadata entries. Later entries with the same key
/// replace earlier ones and the result is ordered by `prompt_order`.
/// - [`ResolvedTranscript`] contains ordered transcript entries after checkpoint application.
/// Transcript preserves order and can be truncated or rewritten by checkpoints.
#[derive(Debug, Clone, PartialEq)]
pub struct ResolvedJournal {
metadata: ResolvedMetadata,
transcript: ResolvedTranscript,
}
impl ResolvedJournal {
fn new(metadata: Vec<JournalEntry>, transcript: Vec<JournalEntry>) -> Self {
Self {
metadata: ResolvedMetadata::new(metadata),
transcript: ResolvedTranscript::new(transcript),
}
}
/// Returns the effective prompt-metadata view.
pub fn metadata(&self) -> &ResolvedMetadata {
&self.metadata
}
/// Returns the effective transcript view.
pub fn transcript(&self) -> &ResolvedTranscript {
&self.transcript
}
/// Returns the effective prompt-metadata view.
pub fn contexts(&self) -> &ResolvedMetadata {
self.metadata()
}
/// Returns the effective transcript view.
pub fn history(&self) -> &ResolvedTranscript {
self.transcript()
}
/// Consumes the resolved view and returns only the prompt-metadata entries.
pub fn into_metadata(self) -> ResolvedMetadata {
self.metadata
}
/// Consumes the resolved view and returns only the transcript entries.
pub fn into_transcript(self) -> ResolvedTranscript {
self.transcript
}
pub fn into_contexts(self) -> ResolvedMetadata {
self.into_metadata()
}
pub fn into_history(self) -> ResolvedTranscript {
self.into_transcript()
}
/// Consumes the resolved view and concatenates metadata followed by transcript entries.
pub fn into_entries(self) -> Vec<JournalEntry> {
self.metadata
.into_entries()
.into_iter()
.chain(self.transcript.into_entries())
.collect()
}
}
/// Effective prompt-metadata entries derived from a journal.
///
/// These entries are deduplicated by key and sorted for prompt rendering. They are suitable for
/// rendering or for building flattened and forked journal states.
#[derive(Debug, Clone, PartialEq)]
pub struct ResolvedMetadata {
entries: Vec<JournalEntry>,
}
impl ResolvedMetadata {
fn new(entries: Vec<JournalEntry>) -> Self {
Self { entries }
}
/// Returns the resolved prompt-metadata entries.
pub fn entries(&self) -> &[JournalEntry] {
&self.entries
}
/// Returns whether there are no resolved prompt-metadata entries.
pub fn is_empty(&self) -> bool {
self.entries.is_empty()
}
/// Returns the number of resolved prompt-metadata entries.
pub fn len(&self) -> usize {
self.entries.len()
}
/// Consumes the view and returns its entries.
pub fn into_entries(self) -> Vec<JournalEntry> {
self.entries
}
}
impl IntoIterator for ResolvedMetadata {
type Item = JournalEntry;
type IntoIter = std::vec::IntoIter<JournalEntry>;
fn into_iter(self) -> Self::IntoIter {
self.entries.into_iter()
}
}
/// Effective transcript entries derived from a journal.
///
/// These entries preserve prompt-visible order after checkpoint application. Unlike prompt
/// metadata, transcript is not deduplicated by key.
#[derive(Debug, Clone, PartialEq)]
pub struct ResolvedTranscript {
entries: Vec<JournalEntry>,
}
impl ResolvedTranscript {
fn new(entries: Vec<JournalEntry>) -> Self {
Self { entries }
}
/// Returns the resolved transcript entries.
pub fn entries(&self) -> &[JournalEntry] {
&self.entries
}
/// Returns whether there are no resolved transcript entries.
pub fn is_empty(&self) -> bool {
self.entries.is_empty()
}
/// Returns the number of resolved transcript entries.
pub fn len(&self) -> usize {
self.entries.len()
}
/// Consumes the view and returns its entries.
pub fn into_entries(self) -> Vec<JournalEntry> {
self.entries
}
}
impl IntoIterator for ResolvedTranscript {
type Item = JournalEntry;
type IntoIter = std::vec::IntoIter<JournalEntry>;
fn into_iter(self) -> Self::IntoIter {
self.entries.into_iter()
}
}
fn apply_checkpoint(
history: &mut Vec<JournalEntry>,
transcript: &mut Vec<JournalEntry>,
checkpoint_key: &JournalKey,
checkpoint: &JournalCheckpointItem,
) -> Result<()> {
@@ -262,40 +462,40 @@ fn apply_checkpoint(
through,
replacement,
}) => {
let keep_from = resolve_cursor(history.as_slice(), through)?;
let mut next_history =
Vec::with_capacity(replacement.len() + history.len().saturating_sub(keep_from));
next_history.extend(
let keep_from = resolve_cursor(transcript.as_slice(), through)?;
let mut next_transcript =
Vec::with_capacity(replacement.len() + transcript.len().saturating_sub(keep_from));
next_transcript.extend(
replacement
.iter()
.cloned()
.enumerate()
.map(|(index, item)| {
JournalEntry::new(
replacement_history_key(checkpoint_key, index, &item),
replacement_transcript_key(checkpoint_key, index, &item),
item,
)
}),
);
next_history.extend(history[keep_from..].iter().cloned());
*history = next_history;
next_transcript.extend(transcript[keep_from..].iter().cloned());
*transcript = next_transcript;
Ok(())
}
JournalCheckpointItem::TruncateHistory(JournalTruncateHistoryCheckpoint { through }) => {
let keep_len = resolve_cursor(history.as_slice(), through)?;
history.truncate(keep_len);
let keep_len = resolve_cursor(transcript.as_slice(), through)?;
transcript.truncate(keep_len);
Ok(())
}
}
}
fn resolve_cursor(history: &[JournalEntry], cursor: &JournalHistoryCursor) -> Result<usize> {
fn resolve_cursor(transcript: &[JournalEntry], cursor: &JournalHistoryCursor) -> Result<usize> {
match cursor {
JournalHistoryCursor::Start => Ok(0),
JournalHistoryCursor::End => Ok(history.len()),
JournalHistoryCursor::AfterItem(history_item_id) => history
JournalHistoryCursor::End => Ok(transcript.len()),
JournalHistoryCursor::AfterItem(history_item_id) => transcript
.iter()
.position(|entry| history_item(entry).id == *history_item_id)
.position(|entry| transcript_item(entry).id == *history_item_id)
.map(|index| index + 1)
.ok_or_else(|| JournalError::UnknownHistoryItemId {
history_item_id: history_item_id.clone(),
@@ -303,32 +503,32 @@ fn resolve_cursor(history: &[JournalEntry], cursor: &JournalHistoryCursor) -> Re
}
}
fn replacement_history_key(
fn replacement_transcript_key(
checkpoint_key: &JournalKey,
index: usize,
history_item: &JournalHistoryItem,
transcript_item: &JournalTranscriptItem,
) -> JournalKey {
checkpoint_key
.child("replacement")
.child(index.to_string())
.child(history_item.id.clone())
.child(transcript_item.id.clone())
}
fn context_item(entry: &JournalEntry) -> &JournalContextItem {
fn metadata_item(entry: &JournalEntry) -> &JournalMetadataItem {
match &entry.item {
JournalItem::Context(item) => item,
_ => unreachable!("resolved context entries must be context items"),
JournalItem::Metadata(item) => item,
_ => unreachable!("resolved metadata entries must be metadata items"),
}
}
fn history_item(entry: &JournalEntry) -> &JournalHistoryItem {
fn transcript_item(entry: &JournalEntry) -> &JournalTranscriptItem {
match &entry.item {
JournalItem::History(item) => item,
_ => unreachable!("resolved history entries must be history items"),
JournalItem::Transcript(item) => item,
_ => unreachable!("resolved transcript entries must be transcript items"),
}
}
fn context_visible_in_view(item: &JournalContextItem, view: &PromptView) -> bool {
fn metadata_visible_in_view(item: &JournalMetadataItem, view: &PromptView) -> bool {
match &item.audience {
JournalContextAudience::All => true,
JournalContextAudience::RootOnly => view.is_root,

27
codex-rs/journal/src/lib.rs
View File

@@ -1,9 +1,20 @@
//! Typed journal model for prompt rendering, filtering, forking, and persistence.
//!
//! A [`Journal`] stores one append-only sequence of [`JournalEntry`] values and resolves it into
//! two derived views:
//!
//! - prompt metadata: keyed, deduplicated entries ordered by `prompt_order`
//! - transcript: ordered prompt-visible items after checkpoint application
//!
//! Callers can resolve the journal once, then choose how to project those views into prompt
//! messages with [`PromptRenderer`].
mod context_builder;
mod error;
pub mod history;
mod journal;
mod prompt_view;
mod render;
#[cfg(test)]
mod tests;
@@ -11,19 +22,29 @@ mod tests;
pub use codex_protocol::journal::JournalCheckpointItem;
pub use codex_protocol::journal::JournalContextAudience;
pub use codex_protocol::journal::JournalContextForkBehavior;
pub use codex_protocol::journal::JournalContextItem;
pub use codex_protocol::journal::JournalContextKey;
pub use codex_protocol::journal::JournalEntry;
pub use codex_protocol::journal::JournalHistoryCursor;
pub use codex_protocol::journal::JournalHistoryItem;
pub use codex_protocol::journal::JournalItem;
pub use codex_protocol::journal::JournalKey;
pub use codex_protocol::journal::JournalMetadataItem;
pub use codex_protocol::journal::JournalReplacePrefixCheckpoint;
pub use codex_protocol::journal::JournalTranscriptItem;
pub use codex_protocol::journal::JournalTruncateHistoryCheckpoint;
pub use codex_protocol::journal::KeyFilter;
pub use codex_protocol::journal::PromptMessage;
pub use codex_protocol::journal::PromptMessageRole;
pub use context_builder::MetadataEntryBuilder;
pub use error::JournalError;
pub use error::Result;
pub use journal::Journal;
pub use journal::ResolvedJournal;
pub use journal::ResolvedMetadata;
pub use journal::ResolvedTranscript;
pub use prompt_view::PromptView;
pub use render::PromptRenderer;
pub type JournalContextItem = JournalMetadataItem;
pub type JournalHistoryItem = JournalTranscriptItem;
pub type ContextEntryBuilder = MetadataEntryBuilder;
pub type ResolvedContexts = ResolvedMetadata;
pub type ResolvedHistory = ResolvedTranscript;

86
codex-rs/journal/src/render.rs Normal file
View File

@@ -0,0 +1,86 @@
use crate::JournalItem;
use crate::KeyFilter;
use crate::PromptMessage;
use crate::PromptMessageRole;
use crate::ResolvedMetadata;
use codex_protocol::models::ContentItem;
use codex_protocol::models::ResponseItem;
/// Renders resolved prompt metadata into model-visible prompt messages.
///
/// Group filters are applied in declaration order. Entries are merged only when they are
/// consecutive, match the same group filter, and share the same role.
#[derive(Debug, Clone, Default)]
pub struct PromptRenderer {
group_filters: Vec<KeyFilter>,
}
impl PromptRenderer {
/// Creates a renderer with no grouping rules.
pub fn new() -> Self {
Self::default()
}
/// Adds one grouping rule for consecutive resolved metadata entries.
pub fn group(mut self, filter: KeyFilter) -> Self {
self.group_filters.push(filter);
self
}
/// Renders resolved prompt metadata according to the configured grouping rules.
pub fn render_metadata(&self, metadata: &ResolvedMetadata) -> Vec<ResponseItem> {
let mut rendered = Vec::new();
let mut current_group: Option<usize> = None;
let mut current_role: Option<PromptMessageRole> = None;
let mut current_content = Vec::new();
for entry in metadata.entries() {
let JournalItem::Metadata(item) = &entry.item else {
continue;
};
let group = self
.group_filters
.iter()
.position(|filter| filter.matches(&entry.key));
if group.is_none() {
flush_prompt_message(&mut rendered, &mut current_role, &mut current_content);
rendered.push(ResponseItem::from(item.clone()));
current_group = None;
continue;
}
if current_group != group || current_role != Some(item.message.role) {
flush_prompt_message(&mut rendered, &mut current_role, &mut current_content);
current_group = group;
current_role = Some(item.message.role);
}
current_content.extend(item.message.content.clone());
}
flush_prompt_message(&mut rendered, &mut current_role, &mut current_content);
rendered
}
pub fn render_contexts(&self, contexts: &ResolvedMetadata) -> Vec<ResponseItem> {
self.render_metadata(contexts)
}
}
fn flush_prompt_message(
rendered: &mut Vec<ResponseItem>,
role: &mut Option<PromptMessageRole>,
content: &mut Vec<ContentItem>,
) {
let Some(role) = role.take() else {
return;
};
if content.is_empty() {
return;
}
rendered.push(ResponseItem::from(PromptMessage::new(
role,
std::mem::take(content),
)));
}

309
codex-rs/journal/src/tests.rs
View File

@@ -2,15 +2,15 @@ use crate::Journal;
use crate::JournalCheckpointItem;
use crate::JournalContextAudience;
use crate::JournalContextForkBehavior;
use crate::JournalContextItem;
use crate::JournalContextKey;
use crate::JournalEntry;
use crate::JournalHistoryCursor;
use crate::JournalHistoryItem;
use crate::JournalMetadataItem;
use crate::JournalReplacePrefixCheckpoint;
use crate::JournalTranscriptItem;
use crate::JournalTruncateHistoryCheckpoint;
use crate::KeyFilter;
use crate::PromptMessage;
use crate::PromptRenderer;
use crate::PromptView;
use codex_protocol::models::ContentItem;
use codex_protocol::models::ResponseItem;
@@ -39,30 +39,21 @@ fn assistant_message(text: &str) -> ResponseItem {
}
}
fn developer_context(
namespace: &str,
name: &str,
text: &str,
prompt_order: i64,
) -> JournalContextItem {
JournalContextItem::new(
JournalContextKey::new(namespace, name, None),
PromptMessage::developer_text(text),
)
.with_prompt_order(prompt_order)
fn developer_context(text: &str, prompt_order: i64) -> JournalMetadataItem {
JournalMetadataItem::new(PromptMessage::developer_text(text)).with_prompt_order(prompt_order)
}
#[test]
fn to_prompt_uses_latest_context_for_key() {
let mut state = Journal::new();
state.add(
["prompt", "permissions", "older"],
developer_context("context", "permissions", "older permissions", 10),
["prompt", "permissions"],
developer_context("older permissions", 10),
);
state.add(["history", "hello"], user_message("hello"));
state.add(
["prompt", "permissions", "newer"],
developer_context("context", "permissions", "newer permissions", 10),
["prompt", "permissions"],
developer_context("newer permissions", 10),
);
let prompt = state
@@ -83,13 +74,11 @@ fn to_prompt_filters_context_by_audience() {
let mut state = Journal::new();
state.add(
["prompt", "root", "hint"],
developer_context("context", "root", "root-only", 0)
.with_audience(JournalContextAudience::RootOnly),
developer_context("root-only", 0).with_audience(JournalContextAudience::RootOnly),
);
state.add(
["prompt", "child", "hint"],
developer_context("context", "child", "child-only", 1)
.with_audience(JournalContextAudience::SubAgentsOnly),
developer_context("child-only", 1).with_audience(JournalContextAudience::SubAgentsOnly),
);
let root_prompt = state
@@ -119,14 +108,8 @@ fn to_prompt_filters_context_by_audience() {
#[test]
fn to_prompt_with_filter_matches_key_prefix() {
let mut state = Journal::new();
state.add(
["prompt", "root", "keep"],
developer_context("context", "keep", "keep me", 0),
);
state.add(
["prompt", "child", "drop"],
developer_context("context", "drop", "drop me", 1),
);
state.add(["prompt", "root", "keep"], developer_context("keep me", 0));
state.add(["prompt", "child", "drop"], developer_context("drop me", 1));
let prompt = state
.to_prompt_with_filter(&PromptView::root(), &KeyFilter::prefix(["prompt", "root"]))
@@ -140,10 +123,10 @@ fn to_prompt_with_filter_matches_key_prefix() {
#[test]
fn checkpoints_replace_prefix_and_then_truncate_history() {
let first = JournalHistoryItem::new(user_message("turn 1"));
let second = JournalHistoryItem::new(assistant_message("turn 1 answer"));
let third = JournalHistoryItem::new(user_message("turn 2"));
let summary = JournalHistoryItem {
let first = JournalTranscriptItem::new(user_message("turn 1"));
let second = JournalTranscriptItem::new(assistant_message("turn 1 answer"));
let third = JournalTranscriptItem::new(user_message("turn 2"));
let summary = JournalTranscriptItem {
id: "summary".to_string(),
turn_id: None,
item: assistant_message("summary"),
@@ -177,22 +160,16 @@ fn checkpoints_replace_prefix_and_then_truncate_history() {
#[test]
fn flatten_preserves_prompt_and_drops_obsolete_items() {
let first = JournalHistoryItem::new(user_message("turn 1"));
let answer = JournalHistoryItem::new(assistant_message("turn 1 answer"));
let summary = JournalHistoryItem {
let first = JournalTranscriptItem::new(user_message("turn 1"));
let answer = JournalTranscriptItem::new(assistant_message("turn 1 answer"));
let summary = JournalTranscriptItem {
id: "summary".to_string(),
turn_id: None,
item: assistant_message("summary"),
};
let state = Journal::from_entries(vec![
JournalEntry::new(
["prompt", "permissions", "old"],
developer_context("context", "permissions", "old", 0),
),
JournalEntry::new(
["prompt", "permissions", "new"],
developer_context("context", "permissions", "new", 0),
),
JournalEntry::new(["prompt", "permissions"], developer_context("old", 0)),
JournalEntry::new(["prompt", "permissions"], developer_context("new", 0)),
JournalEntry::new(["history", "1"], first),
JournalEntry::new(["history", "2"], answer.clone()),
JournalEntry::new(
@@ -216,10 +193,7 @@ fn flatten_preserves_prompt_and_drops_obsolete_items() {
assert_eq!(
flattened.entries(),
vec![
JournalEntry::new(
["prompt", "permissions", "new"],
developer_context("context", "permissions", "new", 0),
),
JournalEntry::new(["prompt", "permissions"], developer_context("new", 0),),
JournalEntry::new(
["checkpoint", "replace", "replacement", "0", "summary"],
summary,
@@ -230,13 +204,13 @@ fn flatten_preserves_prompt_and_drops_obsolete_items() {
#[test]
fn with_history_window_keeps_only_recent_effective_history() {
let first = JournalHistoryItem::new(user_message("turn 1"));
let second = JournalHistoryItem::new(assistant_message("turn 1 answer"));
let third = JournalHistoryItem::new(user_message("turn 2"));
let first = JournalTranscriptItem::new(user_message("turn 1"));
let second = JournalTranscriptItem::new(assistant_message("turn 1 answer"));
let third = JournalTranscriptItem::new(user_message("turn 2"));
let state = Journal::from_entries(vec![
JournalEntry::new(
["prompt", "permissions", "current"],
developer_context("context", "permissions", "p", 0),
developer_context("p", 0),
),
JournalEntry::new(["history", "1"], first),
JournalEntry::new(["history", "2"], second.clone()),
@@ -252,7 +226,7 @@ fn with_history_window_keeps_only_recent_effective_history() {
vec![
JournalEntry::new(
["prompt", "permissions", "current"],
developer_context("context", "permissions", "p", 0),
developer_context("p", 0),
),
JournalEntry::new(["history", "3"], third),
]
@@ -261,23 +235,22 @@ fn with_history_window_keeps_only_recent_effective_history() {
#[test]
fn fork_drops_non_keep_context_and_respects_audience() {
let history = JournalHistoryItem::new(user_message("hello"));
let history = JournalTranscriptItem::new(user_message("hello"));
let state = Journal::from_entries(vec![
JournalEntry::new(
["prompt", "child", "shared"],
developer_context("context", "shared", "shared child context", 0)
developer_context("shared child context", 0)
.with_audience(JournalContextAudience::SubAgentsOnly),
),
JournalEntry::new(
["prompt", "child", "regenerate"],
developer_context("context", "regenerate", "usage hint", 1)
developer_context("usage hint", 1)
.with_audience(JournalContextAudience::SubAgentsOnly)
.with_on_fork(JournalContextForkBehavior::Regenerate),
),
JournalEntry::new(
["prompt", "root", "only"],
developer_context("context", "root", "root only", 2)
.with_audience(JournalContextAudience::RootOnly),
developer_context("root only", 2).with_audience(JournalContextAudience::RootOnly),
),
JournalEntry::new(["history", "hello"], history.clone()),
]);
@@ -294,7 +267,7 @@ fn fork_drops_non_keep_context_and_respects_audience() {
vec![
JournalEntry::new(
["prompt", "child", "shared"],
developer_context("context", "shared", "shared child context", 0)
developer_context("shared child context", 0)
.with_audience(JournalContextAudience::SubAgentsOnly),
),
JournalEntry::new(["history", "hello"], history),
@@ -306,11 +279,11 @@ fn fork_drops_non_keep_context_and_respects_audience() {
fn persist_and_load_jsonl_round_trip() {
let dir = tempdir().expect("tempdir");
let path = dir.path().join("journal.jsonl");
let history = JournalHistoryItem::new(user_message("hello"));
let history = JournalTranscriptItem::new(user_message("hello"));
let state = Journal::from_entries(vec![
JournalEntry::new(
["prompt", "permissions", "current"],
developer_context("context", "permissions", "p", 0),
developer_context("p", 0),
),
JournalEntry::new(["history", "hello"], history),
]);
@@ -326,14 +299,8 @@ fn persist_and_load_jsonl_round_trip() {
#[test]
fn filter_returns_matching_raw_entries() {
let state = Journal::from_entries(vec![
JournalEntry::new(
["prompt", "root", "keep"],
developer_context("context", "keep", "keep me", 0),
),
JournalEntry::new(
["prompt", "child", "drop"],
developer_context("context", "drop", "drop me", 1),
),
JournalEntry::new(["prompt", "root", "keep"], developer_context("keep me", 0)),
JournalEntry::new(["prompt", "child", "drop"], developer_context("drop me", 1)),
JournalEntry::new(["history", "hello"], user_message("hello")),
]);
@@ -343,7 +310,207 @@ fn filter_returns_matching_raw_entries() {
filtered.entries(),
vec![JournalEntry::new(
["prompt", "root", "keep"],
developer_context("context", "keep", "keep me", 0),
developer_context("keep me", 0),
)]
);
}
#[test]
fn context_entry_skips_blank_text_messages() {
assert_eq!(
Journal::context_entry(
["prompt", "developer", "blank"],
10,
PromptMessage::developer_text(" "),
),
None
);
}
#[test]
fn context_entry_builder_carries_optional_fields() {
let entry = Journal::context_entry_builder(
["prompt", "developer", "one"],
PromptMessage::developer_text("first"),
)
.prompt_order(10)
.audience(JournalContextAudience::SubAgentsOnly)
.on_fork(JournalContextForkBehavior::Regenerate)
.tags(vec!["foo".to_string(), "bar".to_string()])
.source("unit-test")
.build()
.expect("entry should be kept");
assert_eq!(
entry,
JournalEntry::new(
["prompt", "developer", "one"],
JournalMetadataItem::new(PromptMessage::developer_text("first"))
.with_prompt_order(10)
.with_audience(JournalContextAudience::SubAgentsOnly)
.with_on_fork(JournalContextForkBehavior::Regenerate)
.with_tags(vec!["foo".to_string(), "bar".to_string()])
.with_source("unit-test"),
)
);
}
#[test]
fn resolve_splits_contexts_from_history() {
let history = JournalTranscriptItem::new(user_message("hello"));
let journal = Journal::from_entries(vec![
Journal::context_entry(
["prompt", "developer", "one"],
10,
PromptMessage::developer_text("first"),
)
.expect("entry should be kept"),
Journal::context_entry(
["prompt", "guardian", "one"],
20,
PromptMessage::developer_text("guardian one"),
)
.expect("entry should be kept"),
Journal::context_entry(
["prompt", "guardian", "two"],
30,
PromptMessage::developer_text("guardian two"),
)
.expect("entry should be kept"),
JournalEntry::new(["history", "hello"], history.clone()),
]);
let resolved = journal.resolve().expect("journal should resolve");
assert_eq!(
resolved.metadata().entries(),
vec![
Journal::context_entry(
["prompt", "developer", "one"],
10,
PromptMessage::developer_text("first"),
)
.expect("entry should be kept"),
Journal::context_entry(
["prompt", "guardian", "one"],
20,
PromptMessage::developer_text("guardian one"),
)
.expect("entry should be kept"),
Journal::context_entry(
["prompt", "guardian", "two"],
30,
PromptMessage::developer_text("guardian two"),
)
.expect("entry should be kept"),
]
);
assert_eq!(
resolved.transcript().entries(),
vec![JournalEntry::new(["history", "hello"], history)]
);
}
#[test]
fn prompt_renderer_groups_by_declared_prefix_and_role() {
let resolved = Journal::from_entries(vec![
Journal::context_entry(
["prompt", "developer", "one"],
10,
PromptMessage::developer_text("first"),
)
.expect("entry should be kept"),
Journal::context_entry(
["prompt", "developer", "two"],
20,
PromptMessage::developer_text("second"),
)
.expect("entry should be kept"),
Journal::context_entry(
["prompt", "contextual_user", "one"],
30,
PromptMessage::user_text("third"),
)
.expect("entry should be kept"),
])
.resolve()
.expect("entries should resolve");
let rendered = PromptRenderer::new()
.group(KeyFilter::prefix(["prompt", "developer"]))
.group(KeyFilter::prefix(["prompt", "contextual_user"]))
.render_metadata(resolved.metadata());
assert_eq!(
rendered,
vec![
ResponseItem::Message {
id: None,
role: "developer".to_string(),
content: vec![
ContentItem::InputText {
text: "first".to_string(),
},
ContentItem::InputText {
text: "second".to_string(),
},
],
phase: None,
},
ResponseItem::Message {
id: None,
role: "user".to_string(),
content: vec![ContentItem::InputText {
text: "third".to_string(),
}],
phase: None,
},
]
);
}
#[test]
fn prompt_renderer_leaves_ungrouped_entries_separate() {
let resolved = Journal::from_entries(vec![
Journal::context_entry(
["prompt", "developer", "one"],
10,
PromptMessage::developer_text("first"),
)
.expect("entry should be kept"),
Journal::context_entry(
["prompt", "guardian", "one"],
20,
PromptMessage::developer_text("guardian one"),
)
.expect("entry should be kept"),
Journal::context_entry(
["prompt", "guardian", "two"],
30,
PromptMessage::developer_text("guardian two"),
)
.expect("entry should be kept"),
])
.resolve()
.expect("entries should resolve");
let rendered = PromptRenderer::new()
.group(KeyFilter::prefix(["prompt", "developer"]))
.render_metadata(resolved.metadata());
assert_eq!(
rendered,
vec![
ResponseItem::Message {
id: None,
role: "developer".to_string(),
content: vec![ContentItem::InputText {
text: "first".to_string(),
}],
phase: None,
},
ResponseItem::from(PromptMessage::developer_text("guardian one")),
ResponseItem::from(PromptMessage::developer_text("guardian two")),
]
);
}

77
codex-rs/protocol/src/journal.rs
View File

@@ -79,30 +79,6 @@ impl KeyFilter {
}
}
/// Stable identity for a prompt-context entry.
#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize, JsonSchema, TS)]
pub struct JournalContextKey {
pub namespace: String,
pub name: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[ts(optional)]
pub instance: Option<String>,
}
impl JournalContextKey {
pub fn new(
namespace: impl Into<String>,
name: impl Into<String>,
instance: Option<String>,
) -> Self {
Self {
namespace: namespace.into(),
name: name.into(),
instance,
}
}
}
/// Minimal message block that can be projected into a model-visible prompt item.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema, TS)]
pub struct PromptMessage {
@@ -170,9 +146,9 @@ impl PromptMessageRole {
}
}
/// Durable history item. Unlike prompt context, history keeps original ordering.
/// Durable transcript item. Unlike prompt metadata, transcript keeps original ordering.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema, TS)]
pub struct JournalHistoryItem {
pub struct JournalTranscriptItem {
pub id: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[ts(optional)]
@@ -180,7 +156,7 @@ pub struct JournalHistoryItem {
pub item: ResponseItem,
}
impl JournalHistoryItem {
impl JournalTranscriptItem {
pub fn new(item: ResponseItem) -> Self {
Self {
id: Uuid::new_v4().to_string(),
@@ -195,28 +171,27 @@ impl JournalHistoryItem {
}
}
impl From<ResponseItem> for JournalHistoryItem {
impl From<ResponseItem> for JournalTranscriptItem {
fn from(value: ResponseItem) -> Self {
Self::new(value)
}
}
impl From<JournalHistoryItem> for ResponseItem {
fn from(value: JournalHistoryItem) -> Self {
impl From<JournalTranscriptItem> for ResponseItem {
fn from(value: JournalTranscriptItem) -> Self {
value.item
}
}
impl From<JournalContextItem> for ResponseItem {
fn from(value: JournalContextItem) -> Self {
impl From<JournalMetadataItem> for ResponseItem {
fn from(value: JournalMetadataItem) -> Self {
value.message.into()
}
}
/// Prompt-context entry with stable identity and filtering metadata.
/// Prompt-metadata entry payload and filtering metadata.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema, TS)]
pub struct JournalContextItem {
pub key: JournalContextKey,
pub struct JournalMetadataItem {
pub message: PromptMessage,
#[serde(default)]
pub prompt_order: i64,
@@ -231,10 +206,9 @@ pub struct JournalContextItem {
pub source: Option<String>,
}
impl JournalContextItem {
pub fn new(key: JournalContextKey, message: PromptMessage) -> Self {
impl JournalMetadataItem {
pub fn new(message: PromptMessage) -> Self {
Self {
key,
message,
prompt_order: 0,
audience: JournalContextAudience::default(),
@@ -306,7 +280,7 @@ pub enum JournalHistoryCursor {
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema, TS)]
pub struct JournalReplacePrefixCheckpoint {
pub through: JournalHistoryCursor,
pub replacement: Vec<JournalHistoryItem>,
pub replacement: Vec<JournalTranscriptItem>,
}
/// Keep only the current history prefix through the resolved cursor.
@@ -327,26 +301,30 @@ pub enum JournalCheckpointItem {
#[serde(tag = "type", content = "payload", rename_all = "snake_case")]
#[ts(tag = "type", content = "payload", rename_all = "snake_case")]
pub enum JournalItem {
History(JournalHistoryItem),
Context(JournalContextItem),
#[serde(rename = "history")]
#[ts(rename = "history")]
Transcript(JournalTranscriptItem),
#[serde(rename = "context")]
#[ts(rename = "context")]
Metadata(JournalMetadataItem),
Checkpoint(JournalCheckpointItem),
}
impl From<ResponseItem> for JournalItem {
fn from(value: ResponseItem) -> Self {
Self::History(value.into())
Self::Transcript(value.into())
}
}
impl From<JournalHistoryItem> for JournalItem {
fn from(value: JournalHistoryItem) -> Self {
Self::History(value)
impl From<JournalTranscriptItem> for JournalItem {
fn from(value: JournalTranscriptItem) -> Self {
Self::Transcript(value)
}
}
impl From<JournalContextItem> for JournalItem {
fn from(value: JournalContextItem) -> Self {
Self::Context(value)
impl From<JournalMetadataItem> for JournalItem {
fn from(value: JournalMetadataItem) -> Self {
Self::Metadata(value)
}
}
@@ -381,3 +359,6 @@ where
Self::new(value.0, value.1)
}
}
pub type JournalHistoryItem = JournalTranscriptItem;
pub type JournalContextItem = JournalMetadataItem;

12
codex-rs/thread-store/src/journal_writer.rs
View File

@@ -229,9 +229,8 @@ fn join_error(err: tokio::task::JoinError) -> ThreadStoreError {
#[cfg(test)]
mod tests {
use codex_journal::Journal;
use codex_journal::JournalContextItem;
use codex_journal::JournalContextKey;
use codex_journal::JournalHistoryItem;
use codex_journal::JournalMetadataItem;
use codex_journal::JournalTranscriptItem;
use codex_journal::PromptMessage;
use codex_protocol::models::ContentItem;
use codex_protocol::models::ResponseItem;
@@ -243,17 +242,14 @@ mod tests {
fn developer_entry(text: &str) -> codex_journal::JournalEntry {
codex_journal::JournalEntry::new(
["prompt", text],
JournalContextItem::new(
JournalContextKey::new("context", text, None),
PromptMessage::developer_text(text),
),
JournalMetadataItem::new(PromptMessage::developer_text(text)),
)
}
fn user_entry(text: &str) -> codex_journal::JournalEntry {
codex_journal::JournalEntry::new(
["history", text],
JournalHistoryItem {
JournalTranscriptItem {
id: format!("history-{text}"),
turn_id: None,
item: ResponseItem::Message {