clone/codex
1
0
Fork 0
mirror of https://github.com/openai/codex.git synced 2026-05-01 18:06:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

[codex] Add local thread store listing (#17824)

Browse Source 
Builds on top of #17659 

Move the filesystem + sqlite thread listing-related operations inside of
a local ThreadStore implementation and call ThreadStore from the places
that used to perform these filesystem/sqlite operations.

This is the first of a series of PRs that will implement the rest of the
local ThreadStore.

Testing:
- added unit tests for the thread store implementation
- adjusted some unit tests in the realtime + personality packages whose
callsites changed. Specifically I'm trying to hide ThreadMetadata inside
of the local implementation and make ThreadMetadata a sqlite
implementation detail concern rather than a public interface, preferring
the more generate StoredThread interface instead
- added a corner case test for the personality migration package that
wasn't covered by the existing test suite
- adjust the behavior of searched thread listing to run the existing
local rollout repair/backfill pass _before_ querying SQLite results, so
callers using ThreadStore::list_threads do not miss matches after a
partial metadata warm-up
This commit is contained in:
Tom
2026-04-15 11:34:27 -07:00
committed by GitHub
parent 78ce61c78e
commit cdfcd2ca92
20 changed files with 821 additions and 233 deletions
Download Patch File Download Diff File Expand all files Collapse all files

187
codex-rs/app-server/src/codex_message_processor.rs
View File

@@ -203,7 +203,6 @@ use codex_chatgpt::connectors;
use codex_cloud_requirements::cloud_requirements_loader;
use codex_config::types::McpServerTransportConfig;
use codex_core::CodexThread;
use codex_core::Cursor as RolloutCursor;
use codex_core::ForkSnapshot;
use codex_core::NewThread;
use codex_core::RolloutRecorder;
@@ -211,7 +210,6 @@ use codex_core::SessionMeta;
use codex_core::SteerInputError;
use codex_core::ThreadConfigSnapshot;
use codex_core::ThreadManager;
use codex_core::ThreadSortKey as CoreThreadSortKey;
use codex_core::append_thread_name;
use codex_core::config::Config;
use codex_core::config::ConfigOverrides;
@@ -232,7 +230,6 @@ use codex_core::find_archived_thread_path_by_id_str;
use codex_core::find_thread_name_by_id;
use codex_core::find_thread_names_by_ids;
use codex_core::find_thread_path_by_id_str;
use codex_core::parse_cursor;
use codex_core::path_utils;
use codex_core::plugins::MarketplaceAddError;
use codex_core::plugins::MarketplaceError;
@@ -322,6 +319,12 @@ use codex_state::StateRuntime;
use codex_state::ThreadMetadata;
use codex_state::ThreadMetadataBuilder;
use codex_state::log_db::LogDbLayer;
use codex_thread_store::ListThreadsParams as StoreListThreadsParams;
use codex_thread_store::LocalThreadStore;
use codex_thread_store::StoredThread;
use codex_thread_store::ThreadSortKey as StoreThreadSortKey;
use codex_thread_store::ThreadStore;
use codex_thread_store::ThreadStoreError;
use codex_utils_absolute_path::AbsolutePathBuf;
use codex_utils_json_to_toml::json_to_toml;
use codex_utils_pty::DEFAULT_OUTPUT_BYTES_CAP;
@@ -3819,15 +3822,15 @@ impl CodexMessageProcessor {
.map(|value| value as usize)
.unwrap_or(THREAD_LIST_DEFAULT_LIMIT)
.clamp(1, THREAD_LIST_MAX_LIMIT);
let core_sort_key = match sort_key.unwrap_or(ThreadSortKey::CreatedAt) {
ThreadSortKey::CreatedAt => CoreThreadSortKey::CreatedAt,
ThreadSortKey::UpdatedAt => CoreThreadSortKey::UpdatedAt,
let store_sort_key = match sort_key.unwrap_or(ThreadSortKey::CreatedAt) {
ThreadSortKey::CreatedAt => StoreThreadSortKey::CreatedAt,
ThreadSortKey::UpdatedAt => StoreThreadSortKey::UpdatedAt,
};
let (summaries, next_cursor) = match self
.list_threads_common(
requested_page_size,
cursor,
core_sort_key,
store_sort_key,
ThreadListFilters {
model_providers,
source_kinds,
@@ -5071,7 +5074,7 @@ impl CodexMessageProcessor {
&self,
requested_page_size: usize,
cursor: Option<String>,
sort_key: CoreThreadSortKey,
sort_key: StoreThreadSortKey,
filters: ThreadListFilters,
) -> Result<(Vec<ConversationSummary>, Option<String>), JSONRPCErrorError> {
let ThreadListFilters {
@@ -5081,16 +5084,7 @@ impl CodexMessageProcessor {
cwd,
search_term,
} = filters;
let mut cursor_obj: Option<RolloutCursor> = match cursor.as_ref() {
Some(cursor_str) => {
Some(parse_cursor(cursor_str).ok_or_else(|| JSONRPCErrorError {
code: INVALID_REQUEST_ERROR_CODE,
message: format!("invalid cursor: {cursor_str}"),
data: None,
})?)
}
None => None,
};
let mut cursor_obj = cursor;
let mut last_cursor = cursor_obj.clone();
let mut remaining = requested_page_size;
let mut items = Vec::with_capacity(requested_page_size);
@@ -5109,54 +5103,26 @@ impl CodexMessageProcessor {
let fallback_provider = self.config.model_provider_id.clone();
let (allowed_sources_vec, source_kind_filter) = compute_source_filters(source_kinds);
let allowed_sources = allowed_sources_vec.as_slice();
let state_db_ctx = get_state_db(&self.config).await;
let store = LocalThreadStore::new(codex_rollout::RolloutConfig::from_view(&self.config));
while remaining > 0 {
let page_size = remaining.min(THREAD_LIST_MAX_LIMIT);
let page = if archived {
RolloutRecorder::list_archived_threads(
&self.config,
let page = store
.list_threads(StoreListThreadsParams {
page_size,
cursor_obj.as_ref(),
cursor: cursor_obj.clone(),
sort_key,
allowed_sources,
model_provider_filter.as_deref(),
fallback_provider.as_str(),
search_term.as_deref(),
)
allowed_sources: allowed_sources.to_vec(),
model_providers: model_provider_filter.clone(),
archived,
search_term: search_term.clone(),
})
.await
.map_err(|err| JSONRPCErrorError {
code: INTERNAL_ERROR_CODE,
message: format!("failed to list threads: {err}"),
data: None,
})?
} else {
RolloutRecorder::list_threads(
&self.config,
page_size,
cursor_obj.as_ref(),
sort_key,
allowed_sources,
model_provider_filter.as_deref(),
fallback_provider.as_str(),
search_term.as_deref(),
)
.await
.map_err(|err| JSONRPCErrorError {
code: INTERNAL_ERROR_CODE,
message: format!("failed to list threads: {err}"),
data: None,
})?
};
.map_err(thread_store_list_error)?;
let mut filtered = Vec::with_capacity(page.items.len());
for it in page.items {
let Some(summary) = summary_from_thread_list_item(
it,
fallback_provider.as_str(),
state_db_ctx.as_ref(),
)
.await
let Some(summary) = summary_from_stored_thread(it, fallback_provider.as_str())
else {
continue;
};
@@ -5176,12 +5142,8 @@ impl CodexMessageProcessor {
items.extend(filtered);
remaining = requested_page_size.saturating_sub(items.len());
// Encode RolloutCursor into the JSON-RPC string form returned to clients.
let next_cursor_value = page.next_cursor.clone();
next_cursor = next_cursor_value
.as_ref()
.and_then(|cursor| serde_json::to_value(cursor).ok())
.and_then(|value| value.as_str().map(str::to_owned));
next_cursor = next_cursor_value.clone();
if remaining == 0 {
break;
}
@@ -9421,67 +9383,52 @@ fn set_thread_name_from_title(thread: &mut Thread, title: String) {
thread.name = Some(title);
}
async fn summary_from_thread_list_item(
it: codex_core::ThreadItem,
fallback_provider: &str,
state_db_ctx: Option<&StateDbHandle>,
) -> Option<ConversationSummary> {
if let Some(thread_id) = it.thread_id {
let timestamp = it.created_at.clone();
let updated_at = it.updated_at.clone().or_else(|| timestamp.clone());
let model_provider = it
.model_provider
.clone()
.unwrap_or_else(|| fallback_provider.to_string());
let cwd = it.cwd?;
let cli_version = it.cli_version.unwrap_or_default();
let source = with_thread_spawn_agent_metadata(
it.source
.unwrap_or(codex_protocol::protocol::SessionSource::Unknown),
it.agent_nickname.clone(),
it.agent_role.clone(),
);
return Some(ConversationSummary {
conversation_id: thread_id,
path: it.path,
preview: it.first_user_message.unwrap_or_default(),
timestamp,
updated_at,
model_provider,
cwd,
cli_version,
source,
git_info: if it.git_sha.is_none()
&& it.git_branch.is_none()
&& it.git_origin_url.is_none()
{
None
} else {
Some(ConversationGitInfo {
sha: it.git_sha,
branch: it.git_branch,
origin_url: it.git_origin_url,
})
},
});
fn thread_store_list_error(err: ThreadStoreError) -> JSONRPCErrorError {
match err {
ThreadStoreError::InvalidRequest { message } => JSONRPCErrorError {
code: INVALID_REQUEST_ERROR_CODE,
message,
data: None,
},
err => JSONRPCErrorError {
code: INTERNAL_ERROR_CODE,
message: format!("failed to list threads: {err}"),
data: None,
},
}
if let Some(thread_id) = thread_id_from_rollout_path(it.path.as_path()) {
return read_summary_from_state_db_context_by_thread_id(state_db_ctx, thread_id).await;
}
None
}
fn thread_id_from_rollout_path(path: &Path) -> Option<ThreadId> {
let file_name = path.file_name()?.to_str()?;
let stem = file_name.strip_suffix(".jsonl")?;
if stem.len() < 37 {
return None;
}
let uuid_start = stem.len().saturating_sub(36);
if !stem[..uuid_start].ends_with('-') {
return None;
}
ThreadId::from_string(&stem[uuid_start..]).ok()
fn summary_from_stored_thread(
thread: StoredThread,
fallback_provider: &str,
) -> Option<ConversationSummary> {
let path = thread.rollout_path?;
let source = with_thread_spawn_agent_metadata(
thread.source,
thread.agent_nickname.clone(),
thread.agent_role.clone(),
);
let git_info = thread.git_info.map(|git| ConversationGitInfo {
sha: git.commit_hash.map(|sha| sha.0),
branch: git.branch,
origin_url: git.repository_url,
});
Some(ConversationSummary {
conversation_id: thread.thread_id,
path,
preview: thread.first_user_message.unwrap_or(thread.preview),
timestamp: Some(thread.created_at.to_rfc3339_opts(SecondsFormat::Secs, true)),
updated_at: Some(thread.updated_at.to_rfc3339_opts(SecondsFormat::Secs, true)),
model_provider: if thread.model_provider.is_empty() {
fallback_provider.to_string()
} else {
thread.model_provider
},
cwd: thread.cwd,
cli_version: thread.cli_version,
source,
git_info,
})
}
#[allow(clippy::too_many_arguments)]