mirror of
https://github.com/openai/codex.git
synced 2026-04-26 23:55:25 +00:00
add @plugin mentions (#13510)
## Note-- added plugin mentions via @, but that conflicts with file mentions depends and builds upon #13433. - introduces explicit `@plugin` mentions. this injects the plugin's mcp servers, app names, and skill name format into turn context as a dev message. - we do not yet have UI for these mentions, so we currently parse raw text (as opposed to skills and apps which have UI chips, autocomplete, etc.) this depends on a `plugins/list` app-server endpoint we can feed the UI with, which is upcoming - also annotate mcp and app tool descriptions with the plugin(s) they come from. this gives the model a first class way of understanding what tools come from which plugins, which will help implicit invocation. ### Tests Added and updated tests, unit and integration. Also confirmed locally a raw `@plugin` injects the dev message, and the model knows about its apps, mcps, and skills.
This commit is contained in:
sayan-oai
@@ -14,7 +14,6 @@ use core_test_support::apps_test_server::AppsTestServer;
|
||||
use core_test_support::responses::ev_completed;
|
||||
use core_test_support::responses::ev_response_created;
|
||||
use core_test_support::responses::mount_sse_once;
|
||||
use core_test_support::responses::mount_sse_sequence;
|
||||
use core_test_support::responses::sse;
|
||||
use core_test_support::responses::start_mock_server;
|
||||
use core_test_support::skip_if_no_network;
|
||||
@@ -22,41 +21,48 @@ use core_test_support::stdio_server_bin;
|
||||
use core_test_support::test_codex::test_codex;
|
||||
use core_test_support::wait_for_event;
|
||||
use core_test_support::wait_for_event_with_timeout;
|
||||
use dunce::canonicalize as normalize_path;
|
||||
use tempfile::TempDir;
|
||||
use wiremock::MockServer;
|
||||
|
||||
fn write_plugin_skill_plugin(home: &TempDir) -> std::path::PathBuf {
|
||||
let plugin_root = home.path().join("plugins/cache/test/sample/local");
|
||||
let skill_dir = plugin_root.join("skills/sample-search");
|
||||
std::fs::create_dir_all(skill_dir.as_path()).expect("create plugin skill dir");
|
||||
const SAMPLE_PLUGIN_CONFIG_NAME: &str = "sample@test";
|
||||
const SAMPLE_PLUGIN_DISPLAY_NAME: &str = "sample";
|
||||
|
||||
fn sample_plugin_root(home: &TempDir) -> std::path::PathBuf {
|
||||
home.path().join("plugins/cache/test/sample/local")
|
||||
}
|
||||
|
||||
fn write_sample_plugin_manifest_and_config(home: &TempDir) -> std::path::PathBuf {
|
||||
let plugin_root = sample_plugin_root(home);
|
||||
std::fs::create_dir_all(plugin_root.join(".codex-plugin")).expect("create plugin manifest dir");
|
||||
std::fs::write(
|
||||
plugin_root.join(".codex-plugin/plugin.json"),
|
||||
r#"{"name":"sample"}"#,
|
||||
format!(r#"{{"name":"{SAMPLE_PLUGIN_DISPLAY_NAME}"}}"#),
|
||||
)
|
||||
.expect("write plugin manifest");
|
||||
std::fs::write(
|
||||
home.path().join("config.toml"),
|
||||
format!(
|
||||
"[features]\nplugins = true\n\n[plugins.\"{SAMPLE_PLUGIN_CONFIG_NAME}\"]\nenabled = true\n"
|
||||
),
|
||||
)
|
||||
.expect("write config");
|
||||
plugin_root
|
||||
}
|
||||
|
||||
fn write_plugin_skill_plugin(home: &TempDir) -> std::path::PathBuf {
|
||||
let plugin_root = write_sample_plugin_manifest_and_config(home);
|
||||
let skill_dir = plugin_root.join("skills/sample-search");
|
||||
std::fs::create_dir_all(skill_dir.as_path()).expect("create plugin skill dir");
|
||||
std::fs::write(
|
||||
skill_dir.join("SKILL.md"),
|
||||
"---\ndescription: inspect sample data\n---\n\n# body\n",
|
||||
)
|
||||
.expect("write plugin skill");
|
||||
std::fs::write(
|
||||
home.path().join("config.toml"),
|
||||
"[features]\nplugins = true\n\n[plugins.\"sample@test\"]\nenabled = true\n",
|
||||
)
|
||||
.expect("write config");
|
||||
skill_dir.join("SKILL.md")
|
||||
}
|
||||
|
||||
fn write_plugin_mcp_plugin(home: &TempDir, command: &str) {
|
||||
let plugin_root = home.path().join("plugins/cache/test/sample/local");
|
||||
std::fs::create_dir_all(plugin_root.join(".codex-plugin")).expect("create plugin manifest dir");
|
||||
std::fs::write(
|
||||
plugin_root.join(".codex-plugin/plugin.json"),
|
||||
r#"{"name":"sample"}"#,
|
||||
)
|
||||
.expect("write plugin manifest");
|
||||
let plugin_root = write_sample_plugin_manifest_and_config(home);
|
||||
std::fs::write(
|
||||
plugin_root.join(".mcp.json"),
|
||||
format!(
|
||||
@@ -70,21 +76,10 @@ fn write_plugin_mcp_plugin(home: &TempDir, command: &str) {
|
||||
),
|
||||
)
|
||||
.expect("write plugin mcp config");
|
||||
std::fs::write(
|
||||
home.path().join("config.toml"),
|
||||
"[features]\nplugins = true\n\n[plugins.\"sample@test\"]\nenabled = true\n",
|
||||
)
|
||||
.expect("write config");
|
||||
}
|
||||
|
||||
fn write_plugin_app_plugin(home: &TempDir) {
|
||||
let plugin_root = home.path().join("plugins/sample");
|
||||
std::fs::create_dir_all(plugin_root.join(".codex-plugin")).expect("create plugin manifest dir");
|
||||
std::fs::write(
|
||||
plugin_root.join(".codex-plugin/plugin.json"),
|
||||
r#"{"name":"sample"}"#,
|
||||
)
|
||||
.expect("write plugin manifest");
|
||||
let plugin_root = write_sample_plugin_manifest_and_config(home);
|
||||
std::fs::write(
|
||||
plugin_root.join(".app.json"),
|
||||
r#"{
|
||||
@@ -96,14 +91,6 @@ fn write_plugin_app_plugin(home: &TempDir) {
|
||||
}"#,
|
||||
)
|
||||
.expect("write plugin app config");
|
||||
std::fs::write(
|
||||
home.path().join("config.toml"),
|
||||
format!(
|
||||
"[features]\nplugins = true\n\n[plugins.sample]\nenabled = true\npath = \"{}\"\n",
|
||||
plugin_root.display()
|
||||
),
|
||||
)
|
||||
.expect("write config");
|
||||
}
|
||||
|
||||
async fn build_plugin_test_codex(
|
||||
@@ -120,6 +107,32 @@ async fn build_plugin_test_codex(
|
||||
.codex)
|
||||
}
|
||||
|
||||
async fn build_apps_enabled_plugin_test_codex(
|
||||
server: &MockServer,
|
||||
codex_home: Arc<TempDir>,
|
||||
chatgpt_base_url: String,
|
||||
) -> Result<Arc<codex_core::CodexThread>> {
|
||||
let mut builder = test_codex()
|
||||
.with_home(codex_home)
|
||||
.with_auth(CodexAuth::from_api_key("Test API Key"))
|
||||
.with_config(move |config| {
|
||||
config
|
||||
.features
|
||||
.enable(Feature::Apps)
|
||||
.expect("test config should allow feature update");
|
||||
config
|
||||
.features
|
||||
.disable(Feature::AppsMcpGateway)
|
||||
.expect("test config should allow feature update");
|
||||
config.chatgpt_base_url = chatgpt_base_url;
|
||||
});
|
||||
Ok(builder
|
||||
.build(server)
|
||||
.await
|
||||
.expect("create new conversation")
|
||||
.codex)
|
||||
}
|
||||
|
||||
fn tool_names(body: &serde_json::Value) -> Vec<String> {
|
||||
body.get("tools")
|
||||
.and_then(serde_json::Value::as_array)
|
||||
@@ -137,6 +150,22 @@ fn tool_names(body: &serde_json::Value) -> Vec<String> {
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
fn tool_description(body: &serde_json::Value, tool_name: &str) -> Option<String> {
|
||||
body.get("tools")
|
||||
.and_then(serde_json::Value::as_array)
|
||||
.and_then(|tools| {
|
||||
tools.iter().find_map(|tool| {
|
||||
if tool.get("name").and_then(serde_json::Value::as_str) == Some(tool_name) {
|
||||
tool.get("description")
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.map(str::to_string)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn plugin_skills_append_to_instructions() -> Result<()> {
|
||||
skip_if_no_network!(Ok(()));
|
||||
@@ -149,7 +178,7 @@ async fn plugin_skills_append_to_instructions() -> Result<()> {
|
||||
.await;
|
||||
|
||||
let codex_home = Arc::new(TempDir::new()?);
|
||||
let skill_path = write_plugin_skill_plugin(codex_home.as_ref());
|
||||
write_plugin_skill_plugin(codex_home.as_ref());
|
||||
let codex = build_plugin_test_codex(&server, Arc::clone(&codex_home)).await?;
|
||||
|
||||
codex
|
||||
@@ -174,76 +203,48 @@ async fn plugin_skills_append_to_instructions() -> Result<()> {
|
||||
"expected plugins section present"
|
||||
);
|
||||
assert!(
|
||||
instructions_text.contains("### Available plugins\n- `sample`"),
|
||||
"expected enabled plugin list in instructions"
|
||||
);
|
||||
assert!(
|
||||
instructions_text.contains("### How to use plugins"),
|
||||
"expected plugin usage guidance heading"
|
||||
);
|
||||
assert!(
|
||||
instructions_text.contains("## Skills"),
|
||||
"expected skills section present"
|
||||
instructions_text.contains("`sample`"),
|
||||
"expected enabled plugin name in instructions"
|
||||
);
|
||||
assert!(
|
||||
instructions_text.contains("sample:sample-search: inspect sample data"),
|
||||
"expected namespaced plugin skill summary"
|
||||
);
|
||||
let expected_path = normalize_path(skill_path)?;
|
||||
let expected_path_str = expected_path.to_string_lossy().replace('\\', "/");
|
||||
assert!(
|
||||
instructions_text.contains(&expected_path_str),
|
||||
"expected path {expected_path_str} in instructions"
|
||||
);
|
||||
assert!(
|
||||
instructions_text.find("## Plugins") < instructions_text.find("## Skills"),
|
||||
"expected plugins section before skills section"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn plugin_apps_expose_tools_after_canonical_name_mention() -> Result<()> {
|
||||
async fn explicit_plugin_mentions_inject_plugin_guidance() -> Result<()> {
|
||||
skip_if_no_network!(Ok(()));
|
||||
let server = start_mock_server().await;
|
||||
let apps_server = AppsTestServer::mount_with_connector_name(&server, "Google Calendar").await?;
|
||||
let mock = mount_sse_sequence(
|
||||
let mock = mount_sse_once(
|
||||
&server,
|
||||
vec![
|
||||
sse(vec![ev_response_created("resp-1"), ev_completed("resp-1")]),
|
||||
sse(vec![ev_response_created("resp-2"), ev_completed("resp-2")]),
|
||||
],
|
||||
sse(vec![ev_response_created("resp-1"), ev_completed("resp-1")]),
|
||||
)
|
||||
.await;
|
||||
|
||||
let codex_home = Arc::new(TempDir::new()?);
|
||||
let rmcp_test_server_bin = match stdio_server_bin() {
|
||||
Ok(bin) => bin,
|
||||
Err(err) => {
|
||||
eprintln!("test_stdio_server binary not available, skipping test: {err}");
|
||||
return Ok(());
|
||||
}
|
||||
};
|
||||
write_plugin_skill_plugin(codex_home.as_ref());
|
||||
write_plugin_mcp_plugin(codex_home.as_ref(), &rmcp_test_server_bin);
|
||||
write_plugin_app_plugin(codex_home.as_ref());
|
||||
#[allow(clippy::expect_used)]
|
||||
let mut builder = test_codex()
|
||||
.with_home(codex_home)
|
||||
.with_auth(CodexAuth::from_api_key("Test API Key"))
|
||||
.with_config(move |config| {
|
||||
config
|
||||
.features
|
||||
.enable(Feature::Apps)
|
||||
.expect("test config should allow feature update");
|
||||
config
|
||||
.features
|
||||
.disable(Feature::AppsMcpGateway)
|
||||
.expect("test config should allow feature update");
|
||||
config.chatgpt_base_url = apps_server.chatgpt_base_url;
|
||||
});
|
||||
let codex = builder
|
||||
.build(&server)
|
||||
.await
|
||||
.expect("create new conversation")
|
||||
.codex;
|
||||
|
||||
let codex =
|
||||
build_apps_enabled_plugin_test_codex(&server, codex_home, apps_server.chatgpt_base_url)
|
||||
.await?;
|
||||
|
||||
codex
|
||||
.submit(Op::UserInput {
|
||||
items: vec![codex_protocol::user_input::UserInput::Text {
|
||||
text: "hello".into(),
|
||||
text: "Use @sample for this task.".into(),
|
||||
text_elements: Vec::new(),
|
||||
}],
|
||||
final_output_json_schema: None,
|
||||
@@ -251,40 +252,46 @@ async fn plugin_apps_expose_tools_after_canonical_name_mention() -> Result<()> {
|
||||
.await?;
|
||||
wait_for_event(&codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await;
|
||||
|
||||
codex
|
||||
.submit(Op::UserInput {
|
||||
items: vec![codex_protocol::user_input::UserInput::Text {
|
||||
text: "Use $google-calendar and then call tools.".into(),
|
||||
text_elements: Vec::new(),
|
||||
}],
|
||||
final_output_json_schema: None,
|
||||
})
|
||||
.await?;
|
||||
wait_for_event(&codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await;
|
||||
|
||||
let requests = mock.requests();
|
||||
assert_eq!(requests.len(), 2, "expected two model requests");
|
||||
|
||||
let first_tools = tool_names(&requests[0].body_json());
|
||||
let request = mock.single_request();
|
||||
let developer_messages = request.message_input_texts("developer");
|
||||
assert!(
|
||||
!first_tools
|
||||
developer_messages
|
||||
.iter()
|
||||
.any(|name| name == "mcp__codex_apps__calendar_create_event"),
|
||||
"app tools should stay hidden before plugin app mention: {first_tools:?}"
|
||||
);
|
||||
|
||||
let second_tools = tool_names(&requests[1].body_json());
|
||||
assert!(
|
||||
second_tools
|
||||
.iter()
|
||||
.any(|name| name == "mcp__codex_apps__calendar_create_event"),
|
||||
"calendar create tool should be available after plugin app mention: {second_tools:?}"
|
||||
.any(|text| text.contains("Skills from this plugin")),
|
||||
"expected plugin skills guidance: {developer_messages:?}"
|
||||
);
|
||||
assert!(
|
||||
second_tools
|
||||
developer_messages
|
||||
.iter()
|
||||
.any(|name| name == "mcp__codex_apps__calendar_list_events"),
|
||||
"calendar list tool should be available after plugin app mention: {second_tools:?}"
|
||||
.any(|text| text.contains("MCP servers from this plugin")),
|
||||
"expected visible plugin MCP guidance: {developer_messages:?}"
|
||||
);
|
||||
assert!(
|
||||
developer_messages
|
||||
.iter()
|
||||
.any(|text| text.contains("Apps from this plugin")),
|
||||
"expected visible plugin app guidance: {developer_messages:?}"
|
||||
);
|
||||
let request_body = request.body_json();
|
||||
let request_tools = tool_names(&request_body);
|
||||
assert!(
|
||||
request_tools
|
||||
.iter()
|
||||
.any(|name| name == "mcp__codex_apps__calendar_create_event"),
|
||||
"expected plugin app tools to become visible for this turn: {request_tools:?}"
|
||||
);
|
||||
let echo_description = tool_description(&request_body, "mcp__sample__echo")
|
||||
.expect("plugin MCP tool description should be present");
|
||||
assert!(
|
||||
echo_description.contains("This tool is part of plugin `sample`."),
|
||||
"expected plugin MCP provenance in tool description: {echo_description:?}"
|
||||
);
|
||||
let calendar_description =
|
||||
tool_description(&request_body, "mcp__codex_apps__calendar_create_event")
|
||||
.expect("plugin app tool description should be present");
|
||||
assert!(
|
||||
calendar_description.contains("This tool is part of plugin `sample`."),
|
||||
"expected plugin app provenance in tool description: {calendar_description:?}"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
|
||||
Reference in New Issue
Block a user