fix: image resizer wasm loading, reenable image resizing (#26805)

This commit is contained in:
Aiden Cline
2026-05-13 23:06:07 -05:00
committed by GitHub
parent c50d2b3656
commit 981e00971a
8 changed files with 419 additions and 66 deletions

View File

@@ -14,7 +14,7 @@ export const Image = Schema.Struct({
description: "Maximum image height before resizing or rejecting the attachment (default: 2000)",
}),
max_base64_bytes: Schema.optional(PositiveInt).annotate({
description: "Maximum base64 payload bytes for an image attachment (default: 4718592)",
description: "Maximum base64 payload bytes for an image attachment (default: 5242880)",
}),
}).annotate({ identifier: "ImageAttachmentConfig" })
export type Image = Schema.Schema.Type<typeof Image>

View File

@@ -1,21 +1,24 @@
import { Config } from "@/config/config"
import type { MessageV2 } from "@/session/message-v2"
import * as Log from "@opencode-ai/core/util/log"
import photonWasm from "@silvia-odwyer/photon-node/photon_rs_bg.wasm" with { type: "file" }
import { Context, Effect, Layer, Schema } from "effect"
import path from "node:path"
import { fileURLToPath } from "node:url"
const MAX_BASE64_BYTES = 4.5 * 1024 * 1024
const MAX_BASE64_BYTES = 5 * 1024 * 1024
const MAX_WIDTH = 2000
const MAX_HEIGHT = 2000
const AUTO_RESIZE = true
const JPEG_QUALITIES = [80, 85, 70, 55, 40]
const log = Log.create({ service: "image" })
export class PhotonUnavailableError extends Schema.TaggedErrorClass<PhotonUnavailableError>()(
"ImagePhotonUnavailableError",
export class ResizerUnavailableError extends Schema.TaggedErrorClass<ResizerUnavailableError>()(
"ImageResizerUnavailableError",
{},
) {
override get message() {
return "Photon image processor is unavailable"
return "Image resizer is unavailable"
}
}
@@ -46,7 +49,7 @@ export class SizeError extends Schema.TaggedErrorClass<SizeError>()("ImageSizeEr
}
}
export type Error = PhotonUnavailableError | InvalidDataUrlError | DecodeError | SizeError
export type Error = ResizerUnavailableError | InvalidDataUrlError | DecodeError | SizeError
export interface Interface {
readonly normalize: (input: MessageV2.FilePart) => Effect.Effect<MessageV2.FilePart, Error>
@@ -59,18 +62,15 @@ export const layer = Layer.effect(
Effect.gen(function* () {
const config = yield* Config.Service
const loadPhoton = yield* Effect.cached(
Effect.promise(async () => {
try {
const photonWasm = (await import("@silvia-odwyer/photon-node/photon_rs_bg.wasm", { with: { type: "file" } }))
.default
// Patched photon-node reads this during module init so Bun compiled binaries use the embedded wasm path.
;(globalThis as typeof globalThis & { __OPENCODE_PHOTON_WASM_PATH?: string }).__OPENCODE_PHOTON_WASM_PATH =
photonWasm
return await import("@silvia-odwyer/photon-node")
} catch {
return null
}
}),
Effect.sync(() => {
// Patched photon-node reads this during module init so Bun compiled binaries use the embedded wasm path.
;(globalThis as typeof globalThis & { __OPENCODE_PHOTON_WASM_PATH?: string }).__OPENCODE_PHOTON_WASM_PATH =
path.isAbsolute(photonWasm) ? photonWasm : fileURLToPath(new URL(photonWasm, import.meta.url))
}).pipe(
Effect.andThen(() => Effect.tryPromise(() => import("@silvia-odwyer/photon-node"))),
Effect.tapError((error) => Effect.sync(() => log.warn("failed to load photon", { error }))),
Effect.mapError(() => new ResizerUnavailableError()),
),
)
const normalize = Effect.fn("Image.normalize")(function* (input: MessageV2.FilePart) {
@@ -85,30 +85,26 @@ export const layer = Layer.effect(
return yield* new InvalidDataUrlError({ url: input.url })
const base64 = input.url.slice(input.url.indexOf(";base64,") + ";base64,".length)
const photon = yield* loadPhoton
if (!photon) return yield* new PhotonUnavailableError()
const bytes = Buffer.byteLength(base64, "utf8")
const decoded = yield* Effect.sync(() => {
try {
return photon.PhotonImage.new_from_byteslice(Buffer.from(base64, "base64"))
} catch {
return undefined
}
const photon = yield* loadPhoton
const decoded = yield* Effect.try({
try: () => photon.PhotonImage.new_from_byteslice(Buffer.from(base64, "base64")),
catch: (error) => {
log.warn("failed to decode image", { error })
return new DecodeError()
},
})
if (!decoded) return yield* new DecodeError()
try {
const originalWidth = decoded.get_width()
const originalHeight = decoded.get_height()
if (
originalWidth <= info.maxWidth &&
originalHeight <= info.maxHeight &&
Buffer.byteLength(base64, "utf8") <= info.maxBase64Bytes
)
if (originalWidth <= info.maxWidth && originalHeight <= info.maxHeight && bytes <= info.maxBase64Bytes)
return input
if (!info.autoResize)
return yield* new SizeError({
bytes: Buffer.byteLength(base64, "utf8"),
bytes,
max: info.maxBase64Bytes,
width: originalWidth,
height: originalHeight,
@@ -159,7 +155,7 @@ export const layer = Layer.effect(
}
return yield* new SizeError({
bytes: Buffer.byteLength(base64, "utf8"),
bytes,
max: info.maxBase64Bytes,
width: originalWidth,
height: originalHeight,

View File

@@ -405,14 +405,18 @@ export const layer: Layer.Layer<
typeof attachment.mime === "string" &&
typeof attachment.url === "string",
)
// temporarily disabled
// const normalized = yield* Effect.forEach(toolAttachments, (attachment) =>
// attachment.mime.startsWith("image/")
// ? image.normalize(attachment).pipe(Effect.exit)
// : Effect.succeed(Exit.succeed<MessageV2.FilePart>(attachment)),
// )
const normalized = yield* Effect.forEach(toolAttachments, (attachment) =>
Effect.succeed(Exit.succeed<MessageV2.FilePart>(attachment)),
attachment.mime.startsWith("image/")
? image
.normalize(attachment)
.pipe(
Effect.catchIf(
(error) => error instanceof Image.ResizerUnavailableError,
() => Effect.succeed(attachment),
),
Effect.exit,
)
: Effect.succeed(Exit.succeed<MessageV2.FilePart>(attachment)),
)
const omitted = normalized.filter(Exit.isFailure).length
const attachments = normalized.filter(Exit.isSuccess).map((item) => item.value)

View File

@@ -42,6 +42,7 @@ import { Shell } from "@/shell/shell"
import { ShellID } from "@/tool/shell/id"
import { AppFileSystem } from "@opencode-ai/core/filesystem"
import { Truncate } from "@/tool/truncate"
import { Image } from "@/image/image"
import { decodeDataUrl } from "@/util/data-url"
import { Process } from "@/util/process"
import { Cause, Effect, Exit, Latch, Layer, Option, Scope, Context, Schema, Types } from "effect"
@@ -165,10 +166,10 @@ function referenceTextPart(input: {
export interface Interface {
readonly cancel: (sessionID: SessionID) => Effect.Effect<void>
readonly prompt: (input: PromptInput) => Effect.Effect<MessageV2.WithParts>
readonly prompt: (input: PromptInput) => Effect.Effect<MessageV2.WithParts, Image.Error>
readonly loop: (input: LoopInput) => Effect.Effect<MessageV2.WithParts>
readonly shell: (input: ShellInput) => Effect.Effect<MessageV2.WithParts, Session.BusyError>
readonly command: (input: CommandInput) => Effect.Effect<MessageV2.WithParts>
readonly command: (input: CommandInput) => Effect.Effect<MessageV2.WithParts, Image.Error>
readonly resolvePromptParts: (template: string) => Effect.Effect<PromptInput["parts"]>
}
@@ -193,6 +194,7 @@ export const layer = Layer.effect(
const lsp = yield* LSP.Service
const registry = yield* ToolRegistry.Service
const truncate = yield* Truncate.Service
const image = yield* Image.Service
const spawner = yield* ChildProcessSpawner.ChildProcessSpawner
const scope = yield* Scope.Scope
const instruction = yield* Instruction.Service
@@ -211,7 +213,7 @@ export const layer = Layer.effect(
return {
cancel: (sessionID: SessionID) => cancel(sessionID),
resolvePromptParts: (template: string) => resolvePromptParts(template),
prompt: (input: PromptInput) => prompt(input),
prompt: (input: PromptInput) => prompt(input).pipe(Effect.catch(Effect.die)),
} satisfies TaskPromptOps
})
@@ -1478,7 +1480,16 @@ NOTE: At any point in time through this workflow you should feel free to ask the
{ message: info, parts: resolvedParts },
)
const parts = resolvedParts
const parts = yield* Effect.forEach(resolvedParts, (part) =>
part.type === "file" && part.mime.startsWith("image/")
? image.normalize(part).pipe(
Effect.catchIf(
(error) => error instanceof Image.ResizerUnavailableError,
() => Effect.succeed(part),
),
)
: Effect.succeed(part),
)
const parsed = decodeMessageInfo(info, { errors: "all", propertyOrder: "original" })
if (Exit.isFailure(parsed)) {
@@ -1599,26 +1610,26 @@ NOTE: At any point in time through this workflow you should feel free to ask the
return { info, parts }
}, Effect.scoped)
const prompt: (input: PromptInput) => Effect.Effect<MessageV2.WithParts> = Effect.fn("SessionPrompt.prompt")(
function* (input: PromptInput) {
const session = yield* sessions.get(input.sessionID).pipe(Effect.orDie)
yield* revert.cleanup(session)
const message = yield* createUserMessage(input)
yield* sessions.touch(input.sessionID)
const prompt: (input: PromptInput) => Effect.Effect<MessageV2.WithParts, Image.Error> = Effect.fn(
"SessionPrompt.prompt",
)(function* (input: PromptInput) {
const session = yield* sessions.get(input.sessionID).pipe(Effect.orDie)
yield* revert.cleanup(session)
const message = yield* createUserMessage(input)
yield* sessions.touch(input.sessionID)
const permissions: Permission.Ruleset = []
for (const [t, enabled] of Object.entries(input.tools ?? {})) {
permissions.push({ permission: t, action: enabled ? "allow" : "deny", pattern: "*" })
}
if (permissions.length > 0) {
session.permission = permissions
yield* sessions.setPermission({ sessionID: session.id, permission: permissions })
}
const permissions: Permission.Ruleset = []
for (const [t, enabled] of Object.entries(input.tools ?? {})) {
permissions.push({ permission: t, action: enabled ? "allow" : "deny", pattern: "*" })
}
if (permissions.length > 0) {
session.permission = permissions
yield* sessions.setPermission({ sessionID: session.id, permission: permissions })
}
if (input.noReply === true) return message
return yield* loop({ sessionID: input.sessionID })
},
)
if (input.noReply === true) return message
return yield* loop({ sessionID: input.sessionID })
})
const lastAssistant = Effect.fnUntraced(function* (sessionID: SessionID) {
const match = yield* sessions.findMessage(sessionID, (m) => m.info.role !== "user").pipe(Effect.orDie)
@@ -2019,6 +2030,7 @@ export const defaultLayer = Layer.suspend(() =>
Layer.provide(Session.defaultLayer),
Layer.provide(SessionRevert.defaultLayer),
Layer.provide(SessionSummary.defaultLayer),
Layer.provide(Image.defaultLayer),
Layer.provide(
Layer.mergeAll(
Agent.defaultLayer,

Binary file not shown.

After

Width:  |  Height:  |  Size: 3.8 MiB

View File

@@ -2,6 +2,7 @@ import { describe, expect } from "bun:test"
import { Cause, Effect, Exit, Layer } from "effect"
import { Image } from "@/image/image"
import { MessageID, PartID, SessionID } from "@/session/schema"
import path from "node:path"
import { TestConfig } from "../fixture/config"
import { testEffect } from "../lib/effect"
@@ -57,6 +58,46 @@ describe("Image", () => {
}),
)
it.effect("resizes images that fit the byte limit but exceed dimension limits", () =>
Effect.gen(function* () {
const photon = yield* Effect.promise(() => import("@silvia-odwyer/photon-node"))
const source = new photon.PhotonImage(new Uint8Array(Array.from({ length: 9_000 * 4 }, () => 255)), 9_000, 1)
const image = yield* Image.Service
const result = yield* image.normalize(part("image/png", Buffer.from(source.get_bytes()).toString("base64")))
const resized = photon.PhotonImage.new_from_byteslice(
Buffer.from(result.url.slice(result.url.indexOf(";base64,") + ";base64,".length), "base64"),
)
source.free()
expect(resized.get_width()).toBeLessThanOrEqual(2_000)
expect(resized.get_height()).toBeLessThanOrEqual(2_000)
resized.free()
}),
)
it.effect("resizes the 5MB base64 picture fixture", () =>
Effect.gen(function* () {
const photon = yield* Effect.promise(() => import("@silvia-odwyer/photon-node"))
const data = Buffer.from(
yield* Effect.promise(() =>
Bun.file(path.join(import.meta.dir, "fixtures", "picture-5mb-base64.png")).arrayBuffer(),
),
)
const input = part("image/png", data.toString("base64"))
const image = yield* Image.Service
const result = yield* image.normalize(input)
const base64 = result.url.slice(result.url.indexOf(";base64,") + ";base64,".length)
const resized = photon.PhotonImage.new_from_byteslice(Buffer.from(base64, "base64"))
expect(input.url.slice(input.url.indexOf(";base64,") + ";base64,".length).length).toBe(5 * 1024 * 1024)
expect(result.url).not.toBe(input.url)
expect(base64.length).toBeLessThan(5 * 1024 * 1024)
expect(resized.get_width()).toBeLessThanOrEqual(2_000)
expect(resized.get_height()).toBeLessThanOrEqual(2_000)
resized.free()
}),
)
tiny.effect("fails with a typed size error when no resized candidate fits", () =>
Effect.gen(function* () {
const photon = yield* Effect.promise(() => import("@silvia-odwyer/photon-node"))

View File

@@ -393,6 +393,35 @@ You can also configure [local models](/docs/models#local). [Learn more](/docs/mo
---
### Image attachments
OpenCode normalizes image attachments before sending them to the model. By default, images are resized when they exceed `2000x2000` pixels or `5242880` base64 bytes.
Configure image attachment limits with the `attachment.image` option:
```json title="opencode.json"
{
"$schema": "https://opencode.ai/config.json",
"attachment": {
"image": {
"auto_resize": true,
"max_width": 2000,
"max_height": 2000,
"max_base64_bytes": 5242880
}
}
}
```
- `auto_resize` - Resize images that exceed the configured limits before provider requests. Set to `false` to reject oversized images instead.
- `max_width` - Maximum image width in pixels before resizing or rejection.
- `max_height` - Maximum image height in pixels before resizing or rejection.
- `max_base64_bytes` - Maximum encoded image payload size. This is the base64 payload size, not the original file size.
If an image still cannot fit after resizing, OpenCode omits oversized tool-result images or fails oversized user-provided images with an image size error.
---
#### Provider-Specific Options
Some providers support additional configuration options beyond the generic `timeout` and `apiKey` settings.