mirror of
https://github.com/openai/codex.git
synced 2026-04-25 07:05:38 +00:00
5ea107a088
Took over the work that @aaronl-openai started here: https://github.com/openai/codex/pull/10397 Now that app-server clients are able to set up custom tools (called `dynamic_tools` in app-server), we should expose a way for clients to pass in not just text, but also image outputs. This is something the Responses API already supports for function call outputs, where you can pass in either a string or an array of content outputs (text, image, file): https://platform.openai.com/docs/api-reference/responses/create#responses_create-input-input_item_list-item-function_tool_call_output-output-array-input_image So let's just plumb it through in Codex (with the caveat that we only support text and image for now). This is implemented end-to-end across app-server v2 protocol types and core tool handling. ## Breaking API change NOTE: This introduces a breaking change with dynamic tools, but I think it's ok since this concept was only recently introduced (https://github.com/openai/codex/pull/9539) and it's better to get the API contract correct. I don't think there are any real consumers of this yet (not even the Codex App). Old shape: `{ "output": "dynamic-ok", "success": true }` New shape: ``` { "contentItems": [ { "type": "inputText", "text": "dynamic-ok" }, { "type": "inputImage", "imageUrl": "data:image/png;base64,AAA" } ] "success": true } ```
936 lines
22 KiB
JSON
936 lines
22 KiB
JSON
{
|
||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||
"definitions": {
|
||
"AskForApproval": {
|
||
"description": "Determines the conditions under which the user is consulted to approve running the command proposed by Codex.",
|
||
"oneOf": [
|
||
{
|
||
"description": "Under this policy, only \"known safe\" commands—as determined by `is_safe_command()`—that **only read files** are auto‑approved. Everything else will ask the user to approve.",
|
||
"enum": [
|
||
"untrusted"
|
||
],
|
||
"type": "string"
|
||
},
|
||
{
|
||
"description": "*All* commands are auto‑approved, but they are expected to run inside a sandbox where network access is disabled and writes are confined to a specific set of paths. If the command fails, it will be escalated to the user to approve execution without a sandbox.",
|
||
"enum": [
|
||
"on-failure"
|
||
],
|
||
"type": "string"
|
||
},
|
||
{
|
||
"description": "The model decides when to ask the user for approval.",
|
||
"enum": [
|
||
"on-request"
|
||
],
|
||
"type": "string"
|
||
},
|
||
{
|
||
"description": "Never ask the user to approve commands. Failures are immediately returned to the model, and never escalated to the user for approval.",
|
||
"enum": [
|
||
"never"
|
||
],
|
||
"type": "string"
|
||
}
|
||
]
|
||
},
|
||
"ContentItem": {
|
||
"oneOf": [
|
||
{
|
||
"properties": {
|
||
"text": {
|
||
"type": "string"
|
||
},
|
||
"type": {
|
||
"enum": [
|
||
"input_text"
|
||
],
|
||
"title": "InputTextContentItemType",
|
||
"type": "string"
|
||
}
|
||
},
|
||
"required": [
|
||
"text",
|
||
"type"
|
||
],
|
||
"title": "InputTextContentItem",
|
||
"type": "object"
|
||
},
|
||
{
|
||
"properties": {
|
||
"image_url": {
|
||
"type": "string"
|
||
},
|
||
"type": {
|
||
"enum": [
|
||
"input_image"
|
||
],
|
||
"title": "InputImageContentItemType",
|
||
"type": "string"
|
||
}
|
||
},
|
||
"required": [
|
||
"image_url",
|
||
"type"
|
||
],
|
||
"title": "InputImageContentItem",
|
||
"type": "object"
|
||
},
|
||
{
|
||
"properties": {
|
||
"text": {
|
||
"type": "string"
|
||
},
|
||
"type": {
|
||
"enum": [
|
||
"output_text"
|
||
],
|
||
"title": "OutputTextContentItemType",
|
||
"type": "string"
|
||
}
|
||
},
|
||
"required": [
|
||
"text",
|
||
"type"
|
||
],
|
||
"title": "OutputTextContentItem",
|
||
"type": "object"
|
||
}
|
||
]
|
||
},
|
||
"FunctionCallOutputBody": {
|
||
"anyOf": [
|
||
{
|
||
"type": "string"
|
||
},
|
||
{
|
||
"items": {
|
||
"$ref": "#/definitions/FunctionCallOutputContentItem"
|
||
},
|
||
"type": "array"
|
||
}
|
||
]
|
||
},
|
||
"FunctionCallOutputContentItem": {
|
||
"description": "Responses API compatible content items that can be returned by a tool call. This is a subset of ContentItem with the types we support as function call outputs.",
|
||
"oneOf": [
|
||
{
|
||
"properties": {
|
||
"text": {
|
||
"type": "string"
|
||
},
|
||
"type": {
|
||
"enum": [
|
||
"input_text"
|
||
],
|
||
"title": "InputTextFunctionCallOutputContentItemType",
|
||
"type": "string"
|
||
}
|
||
},
|
||
"required": [
|
||
"text",
|
||
"type"
|
||
],
|
||
"title": "InputTextFunctionCallOutputContentItem",
|
||
"type": "object"
|
||
},
|
||
{
|
||
"properties": {
|
||
"image_url": {
|
||
"type": "string"
|
||
},
|
||
"type": {
|
||
"enum": [
|
||
"input_image"
|
||
],
|
||
"title": "InputImageFunctionCallOutputContentItemType",
|
||
"type": "string"
|
||
}
|
||
},
|
||
"required": [
|
||
"image_url",
|
||
"type"
|
||
],
|
||
"title": "InputImageFunctionCallOutputContentItem",
|
||
"type": "object"
|
||
}
|
||
]
|
||
},
|
||
"FunctionCallOutputPayload": {
|
||
"description": "The payload we send back to OpenAI when reporting a tool call result.\n\n`body` serializes directly as the wire value for `function_call_output.output`. `success` remains internal metadata for downstream handling.",
|
||
"properties": {
|
||
"body": {
|
||
"$ref": "#/definitions/FunctionCallOutputBody"
|
||
},
|
||
"success": {
|
||
"type": [
|
||
"boolean",
|
||
"null"
|
||
]
|
||
}
|
||
},
|
||
"required": [
|
||
"body"
|
||
],
|
||
"type": "object"
|
||
},
|
||
"GhostCommit": {
|
||
"description": "Details of a ghost commit created from a repository state.",
|
||
"properties": {
|
||
"id": {
|
||
"type": "string"
|
||
},
|
||
"parent": {
|
||
"type": [
|
||
"string",
|
||
"null"
|
||
]
|
||
},
|
||
"preexisting_untracked_dirs": {
|
||
"items": {
|
||
"type": "string"
|
||
},
|
||
"type": "array"
|
||
},
|
||
"preexisting_untracked_files": {
|
||
"items": {
|
||
"type": "string"
|
||
},
|
||
"type": "array"
|
||
}
|
||
},
|
||
"required": [
|
||
"id",
|
||
"preexisting_untracked_dirs",
|
||
"preexisting_untracked_files"
|
||
],
|
||
"type": "object"
|
||
},
|
||
"LocalShellAction": {
|
||
"oneOf": [
|
||
{
|
||
"properties": {
|
||
"command": {
|
||
"items": {
|
||
"type": "string"
|
||
},
|
||
"type": "array"
|
||
},
|
||
"env": {
|
||
"additionalProperties": {
|
||
"type": "string"
|
||
},
|
||
"type": [
|
||
"object",
|
||
"null"
|
||
]
|
||
},
|
||
"timeout_ms": {
|
||
"format": "uint64",
|
||
"minimum": 0.0,
|
||
"type": [
|
||
"integer",
|
||
"null"
|
||
]
|
||
},
|
||
"type": {
|
||
"enum": [
|
||
"exec"
|
||
],
|
||
"title": "ExecLocalShellActionType",
|
||
"type": "string"
|
||
},
|
||
"user": {
|
||
"type": [
|
||
"string",
|
||
"null"
|
||
]
|
||
},
|
||
"working_directory": {
|
||
"type": [
|
||
"string",
|
||
"null"
|
||
]
|
||
}
|
||
},
|
||
"required": [
|
||
"command",
|
||
"type"
|
||
],
|
||
"title": "ExecLocalShellAction",
|
||
"type": "object"
|
||
}
|
||
]
|
||
},
|
||
"LocalShellStatus": {
|
||
"enum": [
|
||
"completed",
|
||
"in_progress",
|
||
"incomplete"
|
||
],
|
||
"type": "string"
|
||
},
|
||
"MessagePhase": {
|
||
"enum": [
|
||
"commentary",
|
||
"final_answer"
|
||
],
|
||
"type": "string"
|
||
},
|
||
"NewConversationParams": {
|
||
"properties": {
|
||
"approvalPolicy": {
|
||
"anyOf": [
|
||
{
|
||
"$ref": "#/definitions/AskForApproval"
|
||
},
|
||
{
|
||
"type": "null"
|
||
}
|
||
]
|
||
},
|
||
"baseInstructions": {
|
||
"type": [
|
||
"string",
|
||
"null"
|
||
]
|
||
},
|
||
"compactPrompt": {
|
||
"type": [
|
||
"string",
|
||
"null"
|
||
]
|
||
},
|
||
"config": {
|
||
"additionalProperties": true,
|
||
"type": [
|
||
"object",
|
||
"null"
|
||
]
|
||
},
|
||
"cwd": {
|
||
"type": [
|
||
"string",
|
||
"null"
|
||
]
|
||
},
|
||
"developerInstructions": {
|
||
"type": [
|
||
"string",
|
||
"null"
|
||
]
|
||
},
|
||
"includeApplyPatchTool": {
|
||
"type": [
|
||
"boolean",
|
||
"null"
|
||
]
|
||
},
|
||
"model": {
|
||
"type": [
|
||
"string",
|
||
"null"
|
||
]
|
||
},
|
||
"modelProvider": {
|
||
"type": [
|
||
"string",
|
||
"null"
|
||
]
|
||
},
|
||
"profile": {
|
||
"type": [
|
||
"string",
|
||
"null"
|
||
]
|
||
},
|
||
"sandbox": {
|
||
"anyOf": [
|
||
{
|
||
"$ref": "#/definitions/SandboxMode"
|
||
},
|
||
{
|
||
"type": "null"
|
||
}
|
||
]
|
||
}
|
||
},
|
||
"type": "object"
|
||
},
|
||
"ReasoningItemContent": {
|
||
"oneOf": [
|
||
{
|
||
"properties": {
|
||
"text": {
|
||
"type": "string"
|
||
},
|
||
"type": {
|
||
"enum": [
|
||
"reasoning_text"
|
||
],
|
||
"title": "ReasoningTextReasoningItemContentType",
|
||
"type": "string"
|
||
}
|
||
},
|
||
"required": [
|
||
"text",
|
||
"type"
|
||
],
|
||
"title": "ReasoningTextReasoningItemContent",
|
||
"type": "object"
|
||
},
|
||
{
|
||
"properties": {
|
||
"text": {
|
||
"type": "string"
|
||
},
|
||
"type": {
|
||
"enum": [
|
||
"text"
|
||
],
|
||
"title": "TextReasoningItemContentType",
|
||
"type": "string"
|
||
}
|
||
},
|
||
"required": [
|
||
"text",
|
||
"type"
|
||
],
|
||
"title": "TextReasoningItemContent",
|
||
"type": "object"
|
||
}
|
||
]
|
||
},
|
||
"ReasoningItemReasoningSummary": {
|
||
"oneOf": [
|
||
{
|
||
"properties": {
|
||
"text": {
|
||
"type": "string"
|
||
},
|
||
"type": {
|
||
"enum": [
|
||
"summary_text"
|
||
],
|
||
"title": "SummaryTextReasoningItemReasoningSummaryType",
|
||
"type": "string"
|
||
}
|
||
},
|
||
"required": [
|
||
"text",
|
||
"type"
|
||
],
|
||
"title": "SummaryTextReasoningItemReasoningSummary",
|
||
"type": "object"
|
||
}
|
||
]
|
||
},
|
||
"ResponseItem": {
|
||
"oneOf": [
|
||
{
|
||
"properties": {
|
||
"content": {
|
||
"items": {
|
||
"$ref": "#/definitions/ContentItem"
|
||
},
|
||
"type": "array"
|
||
},
|
||
"end_turn": {
|
||
"type": [
|
||
"boolean",
|
||
"null"
|
||
]
|
||
},
|
||
"id": {
|
||
"type": [
|
||
"string",
|
||
"null"
|
||
],
|
||
"writeOnly": true
|
||
},
|
||
"phase": {
|
||
"anyOf": [
|
||
{
|
||
"$ref": "#/definitions/MessagePhase"
|
||
},
|
||
{
|
||
"type": "null"
|
||
}
|
||
]
|
||
},
|
||
"role": {
|
||
"type": "string"
|
||
},
|
||
"type": {
|
||
"enum": [
|
||
"message"
|
||
],
|
||
"title": "MessageResponseItemType",
|
||
"type": "string"
|
||
}
|
||
},
|
||
"required": [
|
||
"content",
|
||
"role",
|
||
"type"
|
||
],
|
||
"title": "MessageResponseItem",
|
||
"type": "object"
|
||
},
|
||
{
|
||
"properties": {
|
||
"content": {
|
||
"default": null,
|
||
"items": {
|
||
"$ref": "#/definitions/ReasoningItemContent"
|
||
},
|
||
"type": [
|
||
"array",
|
||
"null"
|
||
]
|
||
},
|
||
"encrypted_content": {
|
||
"type": [
|
||
"string",
|
||
"null"
|
||
]
|
||
},
|
||
"id": {
|
||
"type": "string",
|
||
"writeOnly": true
|
||
},
|
||
"summary": {
|
||
"items": {
|
||
"$ref": "#/definitions/ReasoningItemReasoningSummary"
|
||
},
|
||
"type": "array"
|
||
},
|
||
"type": {
|
||
"enum": [
|
||
"reasoning"
|
||
],
|
||
"title": "ReasoningResponseItemType",
|
||
"type": "string"
|
||
}
|
||
},
|
||
"required": [
|
||
"id",
|
||
"summary",
|
||
"type"
|
||
],
|
||
"title": "ReasoningResponseItem",
|
||
"type": "object"
|
||
},
|
||
{
|
||
"properties": {
|
||
"action": {
|
||
"$ref": "#/definitions/LocalShellAction"
|
||
},
|
||
"call_id": {
|
||
"description": "Set when using the Responses API.",
|
||
"type": [
|
||
"string",
|
||
"null"
|
||
]
|
||
},
|
||
"id": {
|
||
"description": "Legacy id field retained for compatibility with older payloads.",
|
||
"type": [
|
||
"string",
|
||
"null"
|
||
],
|
||
"writeOnly": true
|
||
},
|
||
"status": {
|
||
"$ref": "#/definitions/LocalShellStatus"
|
||
},
|
||
"type": {
|
||
"enum": [
|
||
"local_shell_call"
|
||
],
|
||
"title": "LocalShellCallResponseItemType",
|
||
"type": "string"
|
||
}
|
||
},
|
||
"required": [
|
||
"action",
|
||
"status",
|
||
"type"
|
||
],
|
||
"title": "LocalShellCallResponseItem",
|
||
"type": "object"
|
||
},
|
||
{
|
||
"properties": {
|
||
"arguments": {
|
||
"type": "string"
|
||
},
|
||
"call_id": {
|
||
"type": "string"
|
||
},
|
||
"id": {
|
||
"type": [
|
||
"string",
|
||
"null"
|
||
],
|
||
"writeOnly": true
|
||
},
|
||
"name": {
|
||
"type": "string"
|
||
},
|
||
"type": {
|
||
"enum": [
|
||
"function_call"
|
||
],
|
||
"title": "FunctionCallResponseItemType",
|
||
"type": "string"
|
||
}
|
||
},
|
||
"required": [
|
||
"arguments",
|
||
"call_id",
|
||
"name",
|
||
"type"
|
||
],
|
||
"title": "FunctionCallResponseItem",
|
||
"type": "object"
|
||
},
|
||
{
|
||
"properties": {
|
||
"call_id": {
|
||
"type": "string"
|
||
},
|
||
"output": {
|
||
"$ref": "#/definitions/FunctionCallOutputPayload"
|
||
},
|
||
"type": {
|
||
"enum": [
|
||
"function_call_output"
|
||
],
|
||
"title": "FunctionCallOutputResponseItemType",
|
||
"type": "string"
|
||
}
|
||
},
|
||
"required": [
|
||
"call_id",
|
||
"output",
|
||
"type"
|
||
],
|
||
"title": "FunctionCallOutputResponseItem",
|
||
"type": "object"
|
||
},
|
||
{
|
||
"properties": {
|
||
"call_id": {
|
||
"type": "string"
|
||
},
|
||
"id": {
|
||
"type": [
|
||
"string",
|
||
"null"
|
||
],
|
||
"writeOnly": true
|
||
},
|
||
"input": {
|
||
"type": "string"
|
||
},
|
||
"name": {
|
||
"type": "string"
|
||
},
|
||
"status": {
|
||
"type": [
|
||
"string",
|
||
"null"
|
||
]
|
||
},
|
||
"type": {
|
||
"enum": [
|
||
"custom_tool_call"
|
||
],
|
||
"title": "CustomToolCallResponseItemType",
|
||
"type": "string"
|
||
}
|
||
},
|
||
"required": [
|
||
"call_id",
|
||
"input",
|
||
"name",
|
||
"type"
|
||
],
|
||
"title": "CustomToolCallResponseItem",
|
||
"type": "object"
|
||
},
|
||
{
|
||
"properties": {
|
||
"call_id": {
|
||
"type": "string"
|
||
},
|
||
"output": {
|
||
"type": "string"
|
||
},
|
||
"type": {
|
||
"enum": [
|
||
"custom_tool_call_output"
|
||
],
|
||
"title": "CustomToolCallOutputResponseItemType",
|
||
"type": "string"
|
||
}
|
||
},
|
||
"required": [
|
||
"call_id",
|
||
"output",
|
||
"type"
|
||
],
|
||
"title": "CustomToolCallOutputResponseItem",
|
||
"type": "object"
|
||
},
|
||
{
|
||
"properties": {
|
||
"action": {
|
||
"anyOf": [
|
||
{
|
||
"$ref": "#/definitions/WebSearchAction"
|
||
},
|
||
{
|
||
"type": "null"
|
||
}
|
||
]
|
||
},
|
||
"id": {
|
||
"type": [
|
||
"string",
|
||
"null"
|
||
],
|
||
"writeOnly": true
|
||
},
|
||
"status": {
|
||
"type": [
|
||
"string",
|
||
"null"
|
||
]
|
||
},
|
||
"type": {
|
||
"enum": [
|
||
"web_search_call"
|
||
],
|
||
"title": "WebSearchCallResponseItemType",
|
||
"type": "string"
|
||
}
|
||
},
|
||
"required": [
|
||
"type"
|
||
],
|
||
"title": "WebSearchCallResponseItem",
|
||
"type": "object"
|
||
},
|
||
{
|
||
"properties": {
|
||
"ghost_commit": {
|
||
"$ref": "#/definitions/GhostCommit"
|
||
},
|
||
"type": {
|
||
"enum": [
|
||
"ghost_snapshot"
|
||
],
|
||
"title": "GhostSnapshotResponseItemType",
|
||
"type": "string"
|
||
}
|
||
},
|
||
"required": [
|
||
"ghost_commit",
|
||
"type"
|
||
],
|
||
"title": "GhostSnapshotResponseItem",
|
||
"type": "object"
|
||
},
|
||
{
|
||
"properties": {
|
||
"encrypted_content": {
|
||
"type": "string"
|
||
},
|
||
"type": {
|
||
"enum": [
|
||
"compaction"
|
||
],
|
||
"title": "CompactionResponseItemType",
|
||
"type": "string"
|
||
}
|
||
},
|
||
"required": [
|
||
"encrypted_content",
|
||
"type"
|
||
],
|
||
"title": "CompactionResponseItem",
|
||
"type": "object"
|
||
},
|
||
{
|
||
"properties": {
|
||
"type": {
|
||
"enum": [
|
||
"other"
|
||
],
|
||
"title": "OtherResponseItemType",
|
||
"type": "string"
|
||
}
|
||
},
|
||
"required": [
|
||
"type"
|
||
],
|
||
"title": "OtherResponseItem",
|
||
"type": "object"
|
||
}
|
||
]
|
||
},
|
||
"SandboxMode": {
|
||
"enum": [
|
||
"read-only",
|
||
"workspace-write",
|
||
"danger-full-access"
|
||
],
|
||
"type": "string"
|
||
},
|
||
"ThreadId": {
|
||
"type": "string"
|
||
},
|
||
"WebSearchAction": {
|
||
"oneOf": [
|
||
{
|
||
"properties": {
|
||
"queries": {
|
||
"items": {
|
||
"type": "string"
|
||
},
|
||
"type": [
|
||
"array",
|
||
"null"
|
||
]
|
||
},
|
||
"query": {
|
||
"type": [
|
||
"string",
|
||
"null"
|
||
]
|
||
},
|
||
"type": {
|
||
"enum": [
|
||
"search"
|
||
],
|
||
"title": "SearchWebSearchActionType",
|
||
"type": "string"
|
||
}
|
||
},
|
||
"required": [
|
||
"type"
|
||
],
|
||
"title": "SearchWebSearchAction",
|
||
"type": "object"
|
||
},
|
||
{
|
||
"properties": {
|
||
"type": {
|
||
"enum": [
|
||
"open_page"
|
||
],
|
||
"title": "OpenPageWebSearchActionType",
|
||
"type": "string"
|
||
},
|
||
"url": {
|
||
"type": [
|
||
"string",
|
||
"null"
|
||
]
|
||
}
|
||
},
|
||
"required": [
|
||
"type"
|
||
],
|
||
"title": "OpenPageWebSearchAction",
|
||
"type": "object"
|
||
},
|
||
{
|
||
"properties": {
|
||
"pattern": {
|
||
"type": [
|
||
"string",
|
||
"null"
|
||
]
|
||
},
|
||
"type": {
|
||
"enum": [
|
||
"find_in_page"
|
||
],
|
||
"title": "FindInPageWebSearchActionType",
|
||
"type": "string"
|
||
},
|
||
"url": {
|
||
"type": [
|
||
"string",
|
||
"null"
|
||
]
|
||
}
|
||
},
|
||
"required": [
|
||
"type"
|
||
],
|
||
"title": "FindInPageWebSearchAction",
|
||
"type": "object"
|
||
},
|
||
{
|
||
"properties": {
|
||
"type": {
|
||
"enum": [
|
||
"other"
|
||
],
|
||
"title": "OtherWebSearchActionType",
|
||
"type": "string"
|
||
}
|
||
},
|
||
"required": [
|
||
"type"
|
||
],
|
||
"title": "OtherWebSearchAction",
|
||
"type": "object"
|
||
}
|
||
]
|
||
}
|
||
},
|
||
"properties": {
|
||
"conversationId": {
|
||
"anyOf": [
|
||
{
|
||
"$ref": "#/definitions/ThreadId"
|
||
},
|
||
{
|
||
"type": "null"
|
||
}
|
||
]
|
||
},
|
||
"history": {
|
||
"items": {
|
||
"$ref": "#/definitions/ResponseItem"
|
||
},
|
||
"type": [
|
||
"array",
|
||
"null"
|
||
]
|
||
},
|
||
"overrides": {
|
||
"anyOf": [
|
||
{
|
||
"$ref": "#/definitions/NewConversationParams"
|
||
},
|
||
{
|
||
"type": "null"
|
||
}
|
||
]
|
||
},
|
||
"path": {
|
||
"type": [
|
||
"string",
|
||
"null"
|
||
]
|
||
}
|
||
},
|
||
"title": "ResumeConversationParams",
|
||
"type": "object"
|
||
} |