mirror of
https://github.com/openai/codex.git
synced 2026-04-25 07:05:38 +00:00
5ea107a088
Took over the work that @aaronl-openai started here: https://github.com/openai/codex/pull/10397 Now that app-server clients are able to set up custom tools (called `dynamic_tools` in app-server), we should expose a way for clients to pass in not just text, but also image outputs. This is something the Responses API already supports for function call outputs, where you can pass in either a string or an array of content outputs (text, image, file): https://platform.openai.com/docs/api-reference/responses/create#responses_create-input-input_item_list-item-function_tool_call_output-output-array-input_image So let's just plumb it through in Codex (with the caveat that we only support text and image for now). This is implemented end-to-end across app-server v2 protocol types and core tool handling. ## Breaking API change NOTE: This introduces a breaking change with dynamic tools, but I think it's ok since this concept was only recently introduced (https://github.com/openai/codex/pull/9539) and it's better to get the API contract correct. I don't think there are any real consumers of this yet (not even the Codex App). Old shape: `{ "output": "dynamic-ok", "success": true }` New shape: ``` { "contentItems": [ { "type": "inputText", "text": "dynamic-ok" }, { "type": "inputImage", "imageUrl": "data:image/png;base64,AAA" } ] "success": true } ```
791 lines
18 KiB
JSON
791 lines
18 KiB
JSON
{
|
|
"$schema": "http://json-schema.org/draft-07/schema#",
|
|
"definitions": {
|
|
"ContentItem": {
|
|
"oneOf": [
|
|
{
|
|
"properties": {
|
|
"text": {
|
|
"type": "string"
|
|
},
|
|
"type": {
|
|
"enum": [
|
|
"input_text"
|
|
],
|
|
"title": "InputTextContentItemType",
|
|
"type": "string"
|
|
}
|
|
},
|
|
"required": [
|
|
"text",
|
|
"type"
|
|
],
|
|
"title": "InputTextContentItem",
|
|
"type": "object"
|
|
},
|
|
{
|
|
"properties": {
|
|
"image_url": {
|
|
"type": "string"
|
|
},
|
|
"type": {
|
|
"enum": [
|
|
"input_image"
|
|
],
|
|
"title": "InputImageContentItemType",
|
|
"type": "string"
|
|
}
|
|
},
|
|
"required": [
|
|
"image_url",
|
|
"type"
|
|
],
|
|
"title": "InputImageContentItem",
|
|
"type": "object"
|
|
},
|
|
{
|
|
"properties": {
|
|
"text": {
|
|
"type": "string"
|
|
},
|
|
"type": {
|
|
"enum": [
|
|
"output_text"
|
|
],
|
|
"title": "OutputTextContentItemType",
|
|
"type": "string"
|
|
}
|
|
},
|
|
"required": [
|
|
"text",
|
|
"type"
|
|
],
|
|
"title": "OutputTextContentItem",
|
|
"type": "object"
|
|
}
|
|
]
|
|
},
|
|
"FunctionCallOutputBody": {
|
|
"anyOf": [
|
|
{
|
|
"type": "string"
|
|
},
|
|
{
|
|
"items": {
|
|
"$ref": "#/definitions/FunctionCallOutputContentItem"
|
|
},
|
|
"type": "array"
|
|
}
|
|
]
|
|
},
|
|
"FunctionCallOutputContentItem": {
|
|
"description": "Responses API compatible content items that can be returned by a tool call. This is a subset of ContentItem with the types we support as function call outputs.",
|
|
"oneOf": [
|
|
{
|
|
"properties": {
|
|
"text": {
|
|
"type": "string"
|
|
},
|
|
"type": {
|
|
"enum": [
|
|
"input_text"
|
|
],
|
|
"title": "InputTextFunctionCallOutputContentItemType",
|
|
"type": "string"
|
|
}
|
|
},
|
|
"required": [
|
|
"text",
|
|
"type"
|
|
],
|
|
"title": "InputTextFunctionCallOutputContentItem",
|
|
"type": "object"
|
|
},
|
|
{
|
|
"properties": {
|
|
"image_url": {
|
|
"type": "string"
|
|
},
|
|
"type": {
|
|
"enum": [
|
|
"input_image"
|
|
],
|
|
"title": "InputImageFunctionCallOutputContentItemType",
|
|
"type": "string"
|
|
}
|
|
},
|
|
"required": [
|
|
"image_url",
|
|
"type"
|
|
],
|
|
"title": "InputImageFunctionCallOutputContentItem",
|
|
"type": "object"
|
|
}
|
|
]
|
|
},
|
|
"FunctionCallOutputPayload": {
|
|
"description": "The payload we send back to OpenAI when reporting a tool call result.\n\n`body` serializes directly as the wire value for `function_call_output.output`. `success` remains internal metadata for downstream handling.",
|
|
"properties": {
|
|
"body": {
|
|
"$ref": "#/definitions/FunctionCallOutputBody"
|
|
},
|
|
"success": {
|
|
"type": [
|
|
"boolean",
|
|
"null"
|
|
]
|
|
}
|
|
},
|
|
"required": [
|
|
"body"
|
|
],
|
|
"type": "object"
|
|
},
|
|
"GhostCommit": {
|
|
"description": "Details of a ghost commit created from a repository state.",
|
|
"properties": {
|
|
"id": {
|
|
"type": "string"
|
|
},
|
|
"parent": {
|
|
"type": [
|
|
"string",
|
|
"null"
|
|
]
|
|
},
|
|
"preexisting_untracked_dirs": {
|
|
"items": {
|
|
"type": "string"
|
|
},
|
|
"type": "array"
|
|
},
|
|
"preexisting_untracked_files": {
|
|
"items": {
|
|
"type": "string"
|
|
},
|
|
"type": "array"
|
|
}
|
|
},
|
|
"required": [
|
|
"id",
|
|
"preexisting_untracked_dirs",
|
|
"preexisting_untracked_files"
|
|
],
|
|
"type": "object"
|
|
},
|
|
"LocalShellAction": {
|
|
"oneOf": [
|
|
{
|
|
"properties": {
|
|
"command": {
|
|
"items": {
|
|
"type": "string"
|
|
},
|
|
"type": "array"
|
|
},
|
|
"env": {
|
|
"additionalProperties": {
|
|
"type": "string"
|
|
},
|
|
"type": [
|
|
"object",
|
|
"null"
|
|
]
|
|
},
|
|
"timeout_ms": {
|
|
"format": "uint64",
|
|
"minimum": 0.0,
|
|
"type": [
|
|
"integer",
|
|
"null"
|
|
]
|
|
},
|
|
"type": {
|
|
"enum": [
|
|
"exec"
|
|
],
|
|
"title": "ExecLocalShellActionType",
|
|
"type": "string"
|
|
},
|
|
"user": {
|
|
"type": [
|
|
"string",
|
|
"null"
|
|
]
|
|
},
|
|
"working_directory": {
|
|
"type": [
|
|
"string",
|
|
"null"
|
|
]
|
|
}
|
|
},
|
|
"required": [
|
|
"command",
|
|
"type"
|
|
],
|
|
"title": "ExecLocalShellAction",
|
|
"type": "object"
|
|
}
|
|
]
|
|
},
|
|
"LocalShellStatus": {
|
|
"enum": [
|
|
"completed",
|
|
"in_progress",
|
|
"incomplete"
|
|
],
|
|
"type": "string"
|
|
},
|
|
"MessagePhase": {
|
|
"enum": [
|
|
"commentary",
|
|
"final_answer"
|
|
],
|
|
"type": "string"
|
|
},
|
|
"ReasoningItemContent": {
|
|
"oneOf": [
|
|
{
|
|
"properties": {
|
|
"text": {
|
|
"type": "string"
|
|
},
|
|
"type": {
|
|
"enum": [
|
|
"reasoning_text"
|
|
],
|
|
"title": "ReasoningTextReasoningItemContentType",
|
|
"type": "string"
|
|
}
|
|
},
|
|
"required": [
|
|
"text",
|
|
"type"
|
|
],
|
|
"title": "ReasoningTextReasoningItemContent",
|
|
"type": "object"
|
|
},
|
|
{
|
|
"properties": {
|
|
"text": {
|
|
"type": "string"
|
|
},
|
|
"type": {
|
|
"enum": [
|
|
"text"
|
|
],
|
|
"title": "TextReasoningItemContentType",
|
|
"type": "string"
|
|
}
|
|
},
|
|
"required": [
|
|
"text",
|
|
"type"
|
|
],
|
|
"title": "TextReasoningItemContent",
|
|
"type": "object"
|
|
}
|
|
]
|
|
},
|
|
"ReasoningItemReasoningSummary": {
|
|
"oneOf": [
|
|
{
|
|
"properties": {
|
|
"text": {
|
|
"type": "string"
|
|
},
|
|
"type": {
|
|
"enum": [
|
|
"summary_text"
|
|
],
|
|
"title": "SummaryTextReasoningItemReasoningSummaryType",
|
|
"type": "string"
|
|
}
|
|
},
|
|
"required": [
|
|
"text",
|
|
"type"
|
|
],
|
|
"title": "SummaryTextReasoningItemReasoningSummary",
|
|
"type": "object"
|
|
}
|
|
]
|
|
},
|
|
"ResponseItem": {
|
|
"oneOf": [
|
|
{
|
|
"properties": {
|
|
"content": {
|
|
"items": {
|
|
"$ref": "#/definitions/ContentItem"
|
|
},
|
|
"type": "array"
|
|
},
|
|
"end_turn": {
|
|
"type": [
|
|
"boolean",
|
|
"null"
|
|
]
|
|
},
|
|
"id": {
|
|
"type": [
|
|
"string",
|
|
"null"
|
|
],
|
|
"writeOnly": true
|
|
},
|
|
"phase": {
|
|
"anyOf": [
|
|
{
|
|
"$ref": "#/definitions/MessagePhase"
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
]
|
|
},
|
|
"role": {
|
|
"type": "string"
|
|
},
|
|
"type": {
|
|
"enum": [
|
|
"message"
|
|
],
|
|
"title": "MessageResponseItemType",
|
|
"type": "string"
|
|
}
|
|
},
|
|
"required": [
|
|
"content",
|
|
"role",
|
|
"type"
|
|
],
|
|
"title": "MessageResponseItem",
|
|
"type": "object"
|
|
},
|
|
{
|
|
"properties": {
|
|
"content": {
|
|
"default": null,
|
|
"items": {
|
|
"$ref": "#/definitions/ReasoningItemContent"
|
|
},
|
|
"type": [
|
|
"array",
|
|
"null"
|
|
]
|
|
},
|
|
"encrypted_content": {
|
|
"type": [
|
|
"string",
|
|
"null"
|
|
]
|
|
},
|
|
"id": {
|
|
"type": "string",
|
|
"writeOnly": true
|
|
},
|
|
"summary": {
|
|
"items": {
|
|
"$ref": "#/definitions/ReasoningItemReasoningSummary"
|
|
},
|
|
"type": "array"
|
|
},
|
|
"type": {
|
|
"enum": [
|
|
"reasoning"
|
|
],
|
|
"title": "ReasoningResponseItemType",
|
|
"type": "string"
|
|
}
|
|
},
|
|
"required": [
|
|
"id",
|
|
"summary",
|
|
"type"
|
|
],
|
|
"title": "ReasoningResponseItem",
|
|
"type": "object"
|
|
},
|
|
{
|
|
"properties": {
|
|
"action": {
|
|
"$ref": "#/definitions/LocalShellAction"
|
|
},
|
|
"call_id": {
|
|
"description": "Set when using the Responses API.",
|
|
"type": [
|
|
"string",
|
|
"null"
|
|
]
|
|
},
|
|
"id": {
|
|
"description": "Legacy id field retained for compatibility with older payloads.",
|
|
"type": [
|
|
"string",
|
|
"null"
|
|
],
|
|
"writeOnly": true
|
|
},
|
|
"status": {
|
|
"$ref": "#/definitions/LocalShellStatus"
|
|
},
|
|
"type": {
|
|
"enum": [
|
|
"local_shell_call"
|
|
],
|
|
"title": "LocalShellCallResponseItemType",
|
|
"type": "string"
|
|
}
|
|
},
|
|
"required": [
|
|
"action",
|
|
"status",
|
|
"type"
|
|
],
|
|
"title": "LocalShellCallResponseItem",
|
|
"type": "object"
|
|
},
|
|
{
|
|
"properties": {
|
|
"arguments": {
|
|
"type": "string"
|
|
},
|
|
"call_id": {
|
|
"type": "string"
|
|
},
|
|
"id": {
|
|
"type": [
|
|
"string",
|
|
"null"
|
|
],
|
|
"writeOnly": true
|
|
},
|
|
"name": {
|
|
"type": "string"
|
|
},
|
|
"type": {
|
|
"enum": [
|
|
"function_call"
|
|
],
|
|
"title": "FunctionCallResponseItemType",
|
|
"type": "string"
|
|
}
|
|
},
|
|
"required": [
|
|
"arguments",
|
|
"call_id",
|
|
"name",
|
|
"type"
|
|
],
|
|
"title": "FunctionCallResponseItem",
|
|
"type": "object"
|
|
},
|
|
{
|
|
"properties": {
|
|
"call_id": {
|
|
"type": "string"
|
|
},
|
|
"output": {
|
|
"$ref": "#/definitions/FunctionCallOutputPayload"
|
|
},
|
|
"type": {
|
|
"enum": [
|
|
"function_call_output"
|
|
],
|
|
"title": "FunctionCallOutputResponseItemType",
|
|
"type": "string"
|
|
}
|
|
},
|
|
"required": [
|
|
"call_id",
|
|
"output",
|
|
"type"
|
|
],
|
|
"title": "FunctionCallOutputResponseItem",
|
|
"type": "object"
|
|
},
|
|
{
|
|
"properties": {
|
|
"call_id": {
|
|
"type": "string"
|
|
},
|
|
"id": {
|
|
"type": [
|
|
"string",
|
|
"null"
|
|
],
|
|
"writeOnly": true
|
|
},
|
|
"input": {
|
|
"type": "string"
|
|
},
|
|
"name": {
|
|
"type": "string"
|
|
},
|
|
"status": {
|
|
"type": [
|
|
"string",
|
|
"null"
|
|
]
|
|
},
|
|
"type": {
|
|
"enum": [
|
|
"custom_tool_call"
|
|
],
|
|
"title": "CustomToolCallResponseItemType",
|
|
"type": "string"
|
|
}
|
|
},
|
|
"required": [
|
|
"call_id",
|
|
"input",
|
|
"name",
|
|
"type"
|
|
],
|
|
"title": "CustomToolCallResponseItem",
|
|
"type": "object"
|
|
},
|
|
{
|
|
"properties": {
|
|
"call_id": {
|
|
"type": "string"
|
|
},
|
|
"output": {
|
|
"type": "string"
|
|
},
|
|
"type": {
|
|
"enum": [
|
|
"custom_tool_call_output"
|
|
],
|
|
"title": "CustomToolCallOutputResponseItemType",
|
|
"type": "string"
|
|
}
|
|
},
|
|
"required": [
|
|
"call_id",
|
|
"output",
|
|
"type"
|
|
],
|
|
"title": "CustomToolCallOutputResponseItem",
|
|
"type": "object"
|
|
},
|
|
{
|
|
"properties": {
|
|
"action": {
|
|
"anyOf": [
|
|
{
|
|
"$ref": "#/definitions/WebSearchAction"
|
|
},
|
|
{
|
|
"type": "null"
|
|
}
|
|
]
|
|
},
|
|
"id": {
|
|
"type": [
|
|
"string",
|
|
"null"
|
|
],
|
|
"writeOnly": true
|
|
},
|
|
"status": {
|
|
"type": [
|
|
"string",
|
|
"null"
|
|
]
|
|
},
|
|
"type": {
|
|
"enum": [
|
|
"web_search_call"
|
|
],
|
|
"title": "WebSearchCallResponseItemType",
|
|
"type": "string"
|
|
}
|
|
},
|
|
"required": [
|
|
"type"
|
|
],
|
|
"title": "WebSearchCallResponseItem",
|
|
"type": "object"
|
|
},
|
|
{
|
|
"properties": {
|
|
"ghost_commit": {
|
|
"$ref": "#/definitions/GhostCommit"
|
|
},
|
|
"type": {
|
|
"enum": [
|
|
"ghost_snapshot"
|
|
],
|
|
"title": "GhostSnapshotResponseItemType",
|
|
"type": "string"
|
|
}
|
|
},
|
|
"required": [
|
|
"ghost_commit",
|
|
"type"
|
|
],
|
|
"title": "GhostSnapshotResponseItem",
|
|
"type": "object"
|
|
},
|
|
{
|
|
"properties": {
|
|
"encrypted_content": {
|
|
"type": "string"
|
|
},
|
|
"type": {
|
|
"enum": [
|
|
"compaction"
|
|
],
|
|
"title": "CompactionResponseItemType",
|
|
"type": "string"
|
|
}
|
|
},
|
|
"required": [
|
|
"encrypted_content",
|
|
"type"
|
|
],
|
|
"title": "CompactionResponseItem",
|
|
"type": "object"
|
|
},
|
|
{
|
|
"properties": {
|
|
"type": {
|
|
"enum": [
|
|
"other"
|
|
],
|
|
"title": "OtherResponseItemType",
|
|
"type": "string"
|
|
}
|
|
},
|
|
"required": [
|
|
"type"
|
|
],
|
|
"title": "OtherResponseItem",
|
|
"type": "object"
|
|
}
|
|
]
|
|
},
|
|
"WebSearchAction": {
|
|
"oneOf": [
|
|
{
|
|
"properties": {
|
|
"queries": {
|
|
"items": {
|
|
"type": "string"
|
|
},
|
|
"type": [
|
|
"array",
|
|
"null"
|
|
]
|
|
},
|
|
"query": {
|
|
"type": [
|
|
"string",
|
|
"null"
|
|
]
|
|
},
|
|
"type": {
|
|
"enum": [
|
|
"search"
|
|
],
|
|
"title": "SearchWebSearchActionType",
|
|
"type": "string"
|
|
}
|
|
},
|
|
"required": [
|
|
"type"
|
|
],
|
|
"title": "SearchWebSearchAction",
|
|
"type": "object"
|
|
},
|
|
{
|
|
"properties": {
|
|
"type": {
|
|
"enum": [
|
|
"open_page"
|
|
],
|
|
"title": "OpenPageWebSearchActionType",
|
|
"type": "string"
|
|
},
|
|
"url": {
|
|
"type": [
|
|
"string",
|
|
"null"
|
|
]
|
|
}
|
|
},
|
|
"required": [
|
|
"type"
|
|
],
|
|
"title": "OpenPageWebSearchAction",
|
|
"type": "object"
|
|
},
|
|
{
|
|
"properties": {
|
|
"pattern": {
|
|
"type": [
|
|
"string",
|
|
"null"
|
|
]
|
|
},
|
|
"type": {
|
|
"enum": [
|
|
"find_in_page"
|
|
],
|
|
"title": "FindInPageWebSearchActionType",
|
|
"type": "string"
|
|
},
|
|
"url": {
|
|
"type": [
|
|
"string",
|
|
"null"
|
|
]
|
|
}
|
|
},
|
|
"required": [
|
|
"type"
|
|
],
|
|
"title": "FindInPageWebSearchAction",
|
|
"type": "object"
|
|
},
|
|
{
|
|
"properties": {
|
|
"type": {
|
|
"enum": [
|
|
"other"
|
|
],
|
|
"title": "OtherWebSearchActionType",
|
|
"type": "string"
|
|
}
|
|
},
|
|
"required": [
|
|
"type"
|
|
],
|
|
"title": "OtherWebSearchAction",
|
|
"type": "object"
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"properties": {
|
|
"item": {
|
|
"$ref": "#/definitions/ResponseItem"
|
|
},
|
|
"threadId": {
|
|
"type": "string"
|
|
},
|
|
"turnId": {
|
|
"type": "string"
|
|
}
|
|
},
|
|
"required": [
|
|
"item",
|
|
"threadId",
|
|
"turnId"
|
|
],
|
|
"title": "RawResponseItemCompletedNotification",
|
|
"type": "object"
|
|
} |