From 817cb076a8075314644aa330ce1598cb31c468f3 Mon Sep 17 00:00:00 2001 From: Steffen Deusch Date: Sat, 9 May 2026 18:45:44 +0200 Subject: [PATCH] fix(acp): include tool image attachments in updates (#25128) --- packages/opencode/src/acp/agent.ts | 130 +++++++------ .../test/acp/event-subscription.test.ts | 179 +++++++++++++++++- 2 files changed, 246 insertions(+), 63 deletions(-) diff --git a/packages/opencode/src/acp/agent.ts b/packages/opencode/src/acp/agent.ts index 1d941c6b92..867b830cf2 100644 --- a/packages/opencode/src/acp/agent.ts +++ b/packages/opencode/src/acp/agent.ts @@ -347,33 +347,7 @@ export class Agent implements ACPAgent { this.toolStarts.delete(part.callID) this.shellSnapshots.delete(part.callID) const kind = toToolKind(part.tool) - const content: ToolCallContent[] = [ - { - type: "content", - content: { - type: "text", - text: part.state.output, - }, - }, - ] - - if (kind === "edit") { - const input = part.state.input - const filePath = typeof input["filePath"] === "string" ? input["filePath"] : "" - const oldText = typeof input["oldString"] === "string" ? input["oldString"] : "" - const newText = - typeof input["newString"] === "string" - ? input["newString"] - : typeof input["content"] === "string" - ? input["content"] - : "" - content.push({ - type: "diff", - path: filePath, - oldText, - newText, - }) - } + const content = completedToolContent(part, kind) if (part.tool === "todowrite") { const parsedTodos = decodeTodos(part.state.output) @@ -413,10 +387,7 @@ export class Agent implements ACPAgent { content, title: part.state.title, rawInput: part.state.input, - rawOutput: { - output: part.state.output, - metadata: part.state.metadata, - }, + rawOutput: completedToolRawOutput(part), }, }) .catch((error) => { @@ -860,33 +831,7 @@ export class Agent implements ACPAgent { this.toolStarts.delete(part.callID) this.shellSnapshots.delete(part.callID) const kind = toToolKind(part.tool) - const content: ToolCallContent[] = [ - { - type: "content", - content: { - type: "text", - text: part.state.output, - }, - }, - ] - - if (kind === "edit") { - const input = part.state.input - const filePath = typeof input["filePath"] === "string" ? input["filePath"] : "" - const oldText = typeof input["oldString"] === "string" ? input["oldString"] : "" - const newText = - typeof input["newString"] === "string" - ? input["newString"] - : typeof input["content"] === "string" - ? input["content"] - : "" - content.push({ - type: "diff", - path: filePath, - oldText, - newText, - }) - } + const content = completedToolContent(part, kind) if (part.tool === "todowrite") { const parsedTodos = decodeTodos(part.state.output) @@ -926,10 +871,7 @@ export class Agent implements ACPAgent { content, title: part.state.title, rawInput: part.state.input, - rawOutput: { - output: part.state.output, - metadata: part.state.metadata, - }, + rawOutput: completedToolRawOutput(part), }, }) .catch((err) => { @@ -1655,6 +1597,70 @@ function toLocations(toolName: string, input: Record): { path: stri } } +function completedToolContent(part: ToolPart, kind: ToolKind): ToolCallContent[] { + if (part.state.status !== "completed") return [] + + const content: ToolCallContent[] = [ + { + type: "content", + content: { + type: "text", + text: part.state.output, + }, + }, + ] + + if (kind === "edit") { + const input = part.state.input + const filePath = typeof input["filePath"] === "string" ? input["filePath"] : "" + const oldText = typeof input["oldString"] === "string" ? input["oldString"] : "" + const newText = + typeof input["newString"] === "string" + ? input["newString"] + : typeof input["content"] === "string" + ? input["content"] + : "" + content.push({ + type: "diff", + path: filePath, + oldText, + newText, + }) + } + + content.push(...imageContents(part.state.attachments ?? [])) + return content +} + +function completedToolRawOutput(part: ToolPart) { + if (part.state.status !== "completed") return {} + return { + output: part.state.output, + metadata: part.state.metadata, + ...(part.state.attachments?.length ? { attachments: part.state.attachments } : {}), + } +} + +function imageContents(attachments: Array<{ mime: string; url: string }>): ToolCallContent[] { + return attachments.flatMap((attachment): ToolCallContent[] => { + const match = attachment.url.match(/^data:([^;,]+)(?:;[^,]*)*;base64,(.*)$/) + const mime = match?.[1] ?? attachment.mime + if (!mime.startsWith("image/")) return [] + const data = match?.[2] + if (data === undefined) return [] + return [ + { + type: "content" as const, + content: { + type: "image" as const, + mimeType: mime, + data, + }, + }, + ] + }) +} + async function defaultModel(config: ACPConfig, cwd?: string): Promise<{ providerID: ProviderID; modelID: ModelID }> { const sdk = config.sdk const configured = config.defaultModel diff --git a/packages/opencode/test/acp/event-subscription.test.ts b/packages/opencode/test/acp/event-subscription.test.ts index 2722757ab9..d0fb2e994d 100644 --- a/packages/opencode/test/acp/event-subscription.test.ts +++ b/packages/opencode/test/acp/event-subscription.test.ts @@ -1,7 +1,13 @@ import { describe, expect, test } from "bun:test" import { ACP } from "../../src/acp/agent" import type { AgentSideConnection } from "@agentclientprotocol/sdk" -import type { Event, EventMessagePartUpdated, ToolStatePending, ToolStateRunning } from "@opencode-ai/sdk/v2" +import type { + Event, + EventMessagePartUpdated, + ToolStateCompleted, + ToolStatePending, + ToolStateRunning, +} from "@opencode-ai/sdk/v2" import { Instance } from "../../src/project/instance" import { WithInstance } from "../../src/project/with-instance" import { tmpdir } from "../fixture/fixture" @@ -36,6 +42,14 @@ function isToolCallUpdate( return update.sessionUpdate === "tool_call_update" } +function completedToolUpdate(sessionUpdates: SessionUpdateParams[], sessionId: string, callID: string) { + return sessionUpdates + .filter((u) => u.sessionId === sessionId) + .map((u) => u.update) + .filter(isToolCallUpdate) + .find((u) => u.toolCallId === callID && u.status === "completed") +} + function toolEvent( sessionId: string, cwd: string, @@ -78,6 +92,45 @@ function toolEvent( return { directory: cwd, payload } } +function completedToolEvent( + sessionId: string, + cwd: string, + opts: { + callID: string + tool: string + input: Record + output: string + attachments?: ToolStateCompleted["attachments"] + }, +): GlobalEventEnvelope { + const state: ToolStateCompleted = { + status: "completed", + input: opts.input, + output: opts.output, + title: opts.tool, + metadata: {}, + time: { start: Date.now() - 1, end: Date.now() }, + ...(opts.attachments && { attachments: opts.attachments }), + } + const payload: EventMessagePartUpdated = { + type: "message.part.updated", + properties: { + sessionID: sessionId, + time: Date.now(), + part: { + id: `part_${opts.callID}`, + sessionID: sessionId, + messageID: `msg_${opts.callID}`, + type: "tool", + callID: opts.callID, + tool: opts.tool, + state, + }, + }, + } + return { directory: cwd, payload } +} + function createEventStream() { const queue: GlobalEventEnvelope[] = [] const waiters: Array<(value: GlobalEventEnvelope | undefined) => void> = [] @@ -616,6 +669,130 @@ describe("acp.agent event subscription", () => { }) }) + test("emits image attachments as ACP tool content blocks on live completed tool updates", async () => { + await using tmp = await tmpdir() + await Instance.provide({ + directory: tmp.path, + fn: async () => { + const { agent, controller, sessionUpdates, stop } = createFakeAgent() + const cwd = "/tmp/opencode-acp-test" + const sessionId = await agent.newSession({ cwd, mcpServers: [] } as any).then((x) => x.sessionId) + const data = Buffer.from("image-data").toString("base64") + + controller.push( + completedToolEvent(sessionId, cwd, { + callID: "call_image", + tool: "read", + input: { filePath: "/tmp/image.png" }, + output: "Image read successfully", + attachments: [ + { + id: "part_image", + sessionID: sessionId, + messageID: "msg_image", + type: "file", + mime: "image/png", + filename: "image.png", + url: `data:image/png;base64,${data}`, + }, + { + id: "part_text", + sessionID: sessionId, + messageID: "msg_image", + type: "file", + mime: "text/plain", + filename: "note.txt", + url: "data:text/plain;base64,Zm9v", + }, + ], + }), + ) + await new Promise((r) => setTimeout(r, 20)) + + const update = completedToolUpdate(sessionUpdates, sessionId, "call_image") + expect(update?.content).toContainEqual({ + type: "content", + content: { type: "text", text: "Image read successfully" }, + }) + expect(update?.content).toContainEqual({ + type: "content", + content: { type: "image", mimeType: "image/png", data }, + }) + expect(update?.content?.some((item) => item.type === "content" && item.content.type === "resource")).toBe(false) + expect((update?.rawOutput as { attachments?: unknown[] } | undefined)?.attachments?.length).toBe(2) + + stop() + }, + }) + }) + + test("replays completed tool image attachments as ACP tool content blocks", async () => { + await using tmp = await tmpdir() + await Instance.provide({ + directory: tmp.path, + fn: async () => { + const { agent, sessionUpdates, stop, sdk } = createFakeAgent() + const cwd = "/tmp/opencode-acp-test" + const sessionId = await agent.newSession({ cwd, mcpServers: [] } as any).then((x) => x.sessionId) + const data = Buffer.from("replay-image").toString("base64") + + sdk.session.messages = async () => ({ + data: [ + { + info: { + role: "assistant", + sessionID: sessionId, + }, + parts: [ + { + id: "part_replay", + sessionID: sessionId, + messageID: "msg_replay", + type: "tool", + callID: "call_replay_image", + tool: "webfetch", + state: { + status: "completed", + input: { url: "https://example.com/image.png" }, + output: "Image fetched successfully", + title: "webfetch", + metadata: {}, + time: { start: Date.now() - 1, end: Date.now() }, + attachments: [ + { + id: "part_replay_image", + sessionID: sessionId, + messageID: "msg_replay", + type: "file", + mime: "image/jpeg", + filename: "image.jpg", + url: `data:image/jpeg;base64,${data}`, + }, + ], + }, + }, + ], + }, + ], + }) + + await agent.loadSession({ sessionId, cwd, mcpServers: [] } as any) + + const update = completedToolUpdate(sessionUpdates, sessionId, "call_replay_image") + expect(update?.content).toContainEqual({ + type: "content", + content: { type: "text", text: "Image fetched successfully" }, + }) + expect(update?.content).toContainEqual({ + type: "content", + content: { type: "image", mimeType: "image/jpeg", data }, + }) + + stop() + }, + }) + }) + test("does not emit duplicate synthetic pending after replayed running tool", async () => { await using tmp = await tmpdir() await WithInstance.provide({