mirror of
https://github.com/anomalyco/opencode.git
synced 2026-06-01 19:05:38 +00:00
fix: image resizer wasm loading, reenable image resizing (#26805)
This commit is contained in:
@@ -14,7 +14,7 @@ export const Image = Schema.Struct({
|
||||
description: "Maximum image height before resizing or rejecting the attachment (default: 2000)",
|
||||
}),
|
||||
max_base64_bytes: Schema.optional(PositiveInt).annotate({
|
||||
description: "Maximum base64 payload bytes for an image attachment (default: 4718592)",
|
||||
description: "Maximum base64 payload bytes for an image attachment (default: 5242880)",
|
||||
}),
|
||||
}).annotate({ identifier: "ImageAttachmentConfig" })
|
||||
export type Image = Schema.Schema.Type<typeof Image>
|
||||
|
||||
@@ -1,21 +1,24 @@
|
||||
import { Config } from "@/config/config"
|
||||
import type { MessageV2 } from "@/session/message-v2"
|
||||
import * as Log from "@opencode-ai/core/util/log"
|
||||
import photonWasm from "@silvia-odwyer/photon-node/photon_rs_bg.wasm" with { type: "file" }
|
||||
import { Context, Effect, Layer, Schema } from "effect"
|
||||
import path from "node:path"
|
||||
import { fileURLToPath } from "node:url"
|
||||
|
||||
const MAX_BASE64_BYTES = 4.5 * 1024 * 1024
|
||||
const MAX_BASE64_BYTES = 5 * 1024 * 1024
|
||||
const MAX_WIDTH = 2000
|
||||
const MAX_HEIGHT = 2000
|
||||
const AUTO_RESIZE = true
|
||||
const JPEG_QUALITIES = [80, 85, 70, 55, 40]
|
||||
const log = Log.create({ service: "image" })
|
||||
|
||||
export class PhotonUnavailableError extends Schema.TaggedErrorClass<PhotonUnavailableError>()(
|
||||
"ImagePhotonUnavailableError",
|
||||
export class ResizerUnavailableError extends Schema.TaggedErrorClass<ResizerUnavailableError>()(
|
||||
"ImageResizerUnavailableError",
|
||||
{},
|
||||
) {
|
||||
override get message() {
|
||||
return "Photon image processor is unavailable"
|
||||
return "Image resizer is unavailable"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -46,7 +49,7 @@ export class SizeError extends Schema.TaggedErrorClass<SizeError>()("ImageSizeEr
|
||||
}
|
||||
}
|
||||
|
||||
export type Error = PhotonUnavailableError | InvalidDataUrlError | DecodeError | SizeError
|
||||
export type Error = ResizerUnavailableError | InvalidDataUrlError | DecodeError | SizeError
|
||||
|
||||
export interface Interface {
|
||||
readonly normalize: (input: MessageV2.FilePart) => Effect.Effect<MessageV2.FilePart, Error>
|
||||
@@ -59,18 +62,15 @@ export const layer = Layer.effect(
|
||||
Effect.gen(function* () {
|
||||
const config = yield* Config.Service
|
||||
const loadPhoton = yield* Effect.cached(
|
||||
Effect.promise(async () => {
|
||||
try {
|
||||
const photonWasm = (await import("@silvia-odwyer/photon-node/photon_rs_bg.wasm", { with: { type: "file" } }))
|
||||
.default
|
||||
// Patched photon-node reads this during module init so Bun compiled binaries use the embedded wasm path.
|
||||
;(globalThis as typeof globalThis & { __OPENCODE_PHOTON_WASM_PATH?: string }).__OPENCODE_PHOTON_WASM_PATH =
|
||||
photonWasm
|
||||
return await import("@silvia-odwyer/photon-node")
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}),
|
||||
Effect.sync(() => {
|
||||
// Patched photon-node reads this during module init so Bun compiled binaries use the embedded wasm path.
|
||||
;(globalThis as typeof globalThis & { __OPENCODE_PHOTON_WASM_PATH?: string }).__OPENCODE_PHOTON_WASM_PATH =
|
||||
path.isAbsolute(photonWasm) ? photonWasm : fileURLToPath(new URL(photonWasm, import.meta.url))
|
||||
}).pipe(
|
||||
Effect.andThen(() => Effect.tryPromise(() => import("@silvia-odwyer/photon-node"))),
|
||||
Effect.tapError((error) => Effect.sync(() => log.warn("failed to load photon", { error }))),
|
||||
Effect.mapError(() => new ResizerUnavailableError()),
|
||||
),
|
||||
)
|
||||
|
||||
const normalize = Effect.fn("Image.normalize")(function* (input: MessageV2.FilePart) {
|
||||
@@ -85,30 +85,26 @@ export const layer = Layer.effect(
|
||||
return yield* new InvalidDataUrlError({ url: input.url })
|
||||
|
||||
const base64 = input.url.slice(input.url.indexOf(";base64,") + ";base64,".length)
|
||||
const photon = yield* loadPhoton
|
||||
if (!photon) return yield* new PhotonUnavailableError()
|
||||
const bytes = Buffer.byteLength(base64, "utf8")
|
||||
|
||||
const decoded = yield* Effect.sync(() => {
|
||||
try {
|
||||
return photon.PhotonImage.new_from_byteslice(Buffer.from(base64, "base64"))
|
||||
} catch {
|
||||
return undefined
|
||||
}
|
||||
const photon = yield* loadPhoton
|
||||
|
||||
const decoded = yield* Effect.try({
|
||||
try: () => photon.PhotonImage.new_from_byteslice(Buffer.from(base64, "base64")),
|
||||
catch: (error) => {
|
||||
log.warn("failed to decode image", { error })
|
||||
return new DecodeError()
|
||||
},
|
||||
})
|
||||
if (!decoded) return yield* new DecodeError()
|
||||
|
||||
try {
|
||||
const originalWidth = decoded.get_width()
|
||||
const originalHeight = decoded.get_height()
|
||||
if (
|
||||
originalWidth <= info.maxWidth &&
|
||||
originalHeight <= info.maxHeight &&
|
||||
Buffer.byteLength(base64, "utf8") <= info.maxBase64Bytes
|
||||
)
|
||||
if (originalWidth <= info.maxWidth && originalHeight <= info.maxHeight && bytes <= info.maxBase64Bytes)
|
||||
return input
|
||||
if (!info.autoResize)
|
||||
return yield* new SizeError({
|
||||
bytes: Buffer.byteLength(base64, "utf8"),
|
||||
bytes,
|
||||
max: info.maxBase64Bytes,
|
||||
width: originalWidth,
|
||||
height: originalHeight,
|
||||
@@ -159,7 +155,7 @@ export const layer = Layer.effect(
|
||||
}
|
||||
|
||||
return yield* new SizeError({
|
||||
bytes: Buffer.byteLength(base64, "utf8"),
|
||||
bytes,
|
||||
max: info.maxBase64Bytes,
|
||||
width: originalWidth,
|
||||
height: originalHeight,
|
||||
|
||||
@@ -405,14 +405,18 @@ export const layer: Layer.Layer<
|
||||
typeof attachment.mime === "string" &&
|
||||
typeof attachment.url === "string",
|
||||
)
|
||||
// temporarily disabled
|
||||
// const normalized = yield* Effect.forEach(toolAttachments, (attachment) =>
|
||||
// attachment.mime.startsWith("image/")
|
||||
// ? image.normalize(attachment).pipe(Effect.exit)
|
||||
// : Effect.succeed(Exit.succeed<MessageV2.FilePart>(attachment)),
|
||||
// )
|
||||
const normalized = yield* Effect.forEach(toolAttachments, (attachment) =>
|
||||
Effect.succeed(Exit.succeed<MessageV2.FilePart>(attachment)),
|
||||
attachment.mime.startsWith("image/")
|
||||
? image
|
||||
.normalize(attachment)
|
||||
.pipe(
|
||||
Effect.catchIf(
|
||||
(error) => error instanceof Image.ResizerUnavailableError,
|
||||
() => Effect.succeed(attachment),
|
||||
),
|
||||
Effect.exit,
|
||||
)
|
||||
: Effect.succeed(Exit.succeed<MessageV2.FilePart>(attachment)),
|
||||
)
|
||||
const omitted = normalized.filter(Exit.isFailure).length
|
||||
const attachments = normalized.filter(Exit.isSuccess).map((item) => item.value)
|
||||
|
||||
@@ -42,6 +42,7 @@ import { Shell } from "@/shell/shell"
|
||||
import { ShellID } from "@/tool/shell/id"
|
||||
import { AppFileSystem } from "@opencode-ai/core/filesystem"
|
||||
import { Truncate } from "@/tool/truncate"
|
||||
import { Image } from "@/image/image"
|
||||
import { decodeDataUrl } from "@/util/data-url"
|
||||
import { Process } from "@/util/process"
|
||||
import { Cause, Effect, Exit, Latch, Layer, Option, Scope, Context, Schema, Types } from "effect"
|
||||
@@ -165,10 +166,10 @@ function referenceTextPart(input: {
|
||||
|
||||
export interface Interface {
|
||||
readonly cancel: (sessionID: SessionID) => Effect.Effect<void>
|
||||
readonly prompt: (input: PromptInput) => Effect.Effect<MessageV2.WithParts>
|
||||
readonly prompt: (input: PromptInput) => Effect.Effect<MessageV2.WithParts, Image.Error>
|
||||
readonly loop: (input: LoopInput) => Effect.Effect<MessageV2.WithParts>
|
||||
readonly shell: (input: ShellInput) => Effect.Effect<MessageV2.WithParts, Session.BusyError>
|
||||
readonly command: (input: CommandInput) => Effect.Effect<MessageV2.WithParts>
|
||||
readonly command: (input: CommandInput) => Effect.Effect<MessageV2.WithParts, Image.Error>
|
||||
readonly resolvePromptParts: (template: string) => Effect.Effect<PromptInput["parts"]>
|
||||
}
|
||||
|
||||
@@ -193,6 +194,7 @@ export const layer = Layer.effect(
|
||||
const lsp = yield* LSP.Service
|
||||
const registry = yield* ToolRegistry.Service
|
||||
const truncate = yield* Truncate.Service
|
||||
const image = yield* Image.Service
|
||||
const spawner = yield* ChildProcessSpawner.ChildProcessSpawner
|
||||
const scope = yield* Scope.Scope
|
||||
const instruction = yield* Instruction.Service
|
||||
@@ -211,7 +213,7 @@ export const layer = Layer.effect(
|
||||
return {
|
||||
cancel: (sessionID: SessionID) => cancel(sessionID),
|
||||
resolvePromptParts: (template: string) => resolvePromptParts(template),
|
||||
prompt: (input: PromptInput) => prompt(input),
|
||||
prompt: (input: PromptInput) => prompt(input).pipe(Effect.catch(Effect.die)),
|
||||
} satisfies TaskPromptOps
|
||||
})
|
||||
|
||||
@@ -1478,7 +1480,16 @@ NOTE: At any point in time through this workflow you should feel free to ask the
|
||||
{ message: info, parts: resolvedParts },
|
||||
)
|
||||
|
||||
const parts = resolvedParts
|
||||
const parts = yield* Effect.forEach(resolvedParts, (part) =>
|
||||
part.type === "file" && part.mime.startsWith("image/")
|
||||
? image.normalize(part).pipe(
|
||||
Effect.catchIf(
|
||||
(error) => error instanceof Image.ResizerUnavailableError,
|
||||
() => Effect.succeed(part),
|
||||
),
|
||||
)
|
||||
: Effect.succeed(part),
|
||||
)
|
||||
|
||||
const parsed = decodeMessageInfo(info, { errors: "all", propertyOrder: "original" })
|
||||
if (Exit.isFailure(parsed)) {
|
||||
@@ -1599,26 +1610,26 @@ NOTE: At any point in time through this workflow you should feel free to ask the
|
||||
return { info, parts }
|
||||
}, Effect.scoped)
|
||||
|
||||
const prompt: (input: PromptInput) => Effect.Effect<MessageV2.WithParts> = Effect.fn("SessionPrompt.prompt")(
|
||||
function* (input: PromptInput) {
|
||||
const session = yield* sessions.get(input.sessionID).pipe(Effect.orDie)
|
||||
yield* revert.cleanup(session)
|
||||
const message = yield* createUserMessage(input)
|
||||
yield* sessions.touch(input.sessionID)
|
||||
const prompt: (input: PromptInput) => Effect.Effect<MessageV2.WithParts, Image.Error> = Effect.fn(
|
||||
"SessionPrompt.prompt",
|
||||
)(function* (input: PromptInput) {
|
||||
const session = yield* sessions.get(input.sessionID).pipe(Effect.orDie)
|
||||
yield* revert.cleanup(session)
|
||||
const message = yield* createUserMessage(input)
|
||||
yield* sessions.touch(input.sessionID)
|
||||
|
||||
const permissions: Permission.Ruleset = []
|
||||
for (const [t, enabled] of Object.entries(input.tools ?? {})) {
|
||||
permissions.push({ permission: t, action: enabled ? "allow" : "deny", pattern: "*" })
|
||||
}
|
||||
if (permissions.length > 0) {
|
||||
session.permission = permissions
|
||||
yield* sessions.setPermission({ sessionID: session.id, permission: permissions })
|
||||
}
|
||||
const permissions: Permission.Ruleset = []
|
||||
for (const [t, enabled] of Object.entries(input.tools ?? {})) {
|
||||
permissions.push({ permission: t, action: enabled ? "allow" : "deny", pattern: "*" })
|
||||
}
|
||||
if (permissions.length > 0) {
|
||||
session.permission = permissions
|
||||
yield* sessions.setPermission({ sessionID: session.id, permission: permissions })
|
||||
}
|
||||
|
||||
if (input.noReply === true) return message
|
||||
return yield* loop({ sessionID: input.sessionID })
|
||||
},
|
||||
)
|
||||
if (input.noReply === true) return message
|
||||
return yield* loop({ sessionID: input.sessionID })
|
||||
})
|
||||
|
||||
const lastAssistant = Effect.fnUntraced(function* (sessionID: SessionID) {
|
||||
const match = yield* sessions.findMessage(sessionID, (m) => m.info.role !== "user").pipe(Effect.orDie)
|
||||
@@ -2019,6 +2030,7 @@ export const defaultLayer = Layer.suspend(() =>
|
||||
Layer.provide(Session.defaultLayer),
|
||||
Layer.provide(SessionRevert.defaultLayer),
|
||||
Layer.provide(SessionSummary.defaultLayer),
|
||||
Layer.provide(Image.defaultLayer),
|
||||
Layer.provide(
|
||||
Layer.mergeAll(
|
||||
Agent.defaultLayer,
|
||||
|
||||
BIN
packages/opencode/test/image/fixtures/picture-5mb-base64.png
Normal file
BIN
packages/opencode/test/image/fixtures/picture-5mb-base64.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 3.8 MiB |
@@ -2,6 +2,7 @@ import { describe, expect } from "bun:test"
|
||||
import { Cause, Effect, Exit, Layer } from "effect"
|
||||
import { Image } from "@/image/image"
|
||||
import { MessageID, PartID, SessionID } from "@/session/schema"
|
||||
import path from "node:path"
|
||||
import { TestConfig } from "../fixture/config"
|
||||
import { testEffect } from "../lib/effect"
|
||||
|
||||
@@ -57,6 +58,46 @@ describe("Image", () => {
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("resizes images that fit the byte limit but exceed dimension limits", () =>
|
||||
Effect.gen(function* () {
|
||||
const photon = yield* Effect.promise(() => import("@silvia-odwyer/photon-node"))
|
||||
const source = new photon.PhotonImage(new Uint8Array(Array.from({ length: 9_000 * 4 }, () => 255)), 9_000, 1)
|
||||
const image = yield* Image.Service
|
||||
const result = yield* image.normalize(part("image/png", Buffer.from(source.get_bytes()).toString("base64")))
|
||||
const resized = photon.PhotonImage.new_from_byteslice(
|
||||
Buffer.from(result.url.slice(result.url.indexOf(";base64,") + ";base64,".length), "base64"),
|
||||
)
|
||||
|
||||
source.free()
|
||||
expect(resized.get_width()).toBeLessThanOrEqual(2_000)
|
||||
expect(resized.get_height()).toBeLessThanOrEqual(2_000)
|
||||
resized.free()
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("resizes the 5MB base64 picture fixture", () =>
|
||||
Effect.gen(function* () {
|
||||
const photon = yield* Effect.promise(() => import("@silvia-odwyer/photon-node"))
|
||||
const data = Buffer.from(
|
||||
yield* Effect.promise(() =>
|
||||
Bun.file(path.join(import.meta.dir, "fixtures", "picture-5mb-base64.png")).arrayBuffer(),
|
||||
),
|
||||
)
|
||||
const input = part("image/png", data.toString("base64"))
|
||||
const image = yield* Image.Service
|
||||
const result = yield* image.normalize(input)
|
||||
const base64 = result.url.slice(result.url.indexOf(";base64,") + ";base64,".length)
|
||||
const resized = photon.PhotonImage.new_from_byteslice(Buffer.from(base64, "base64"))
|
||||
|
||||
expect(input.url.slice(input.url.indexOf(";base64,") + ";base64,".length).length).toBe(5 * 1024 * 1024)
|
||||
expect(result.url).not.toBe(input.url)
|
||||
expect(base64.length).toBeLessThan(5 * 1024 * 1024)
|
||||
expect(resized.get_width()).toBeLessThanOrEqual(2_000)
|
||||
expect(resized.get_height()).toBeLessThanOrEqual(2_000)
|
||||
resized.free()
|
||||
}),
|
||||
)
|
||||
|
||||
tiny.effect("fails with a typed size error when no resized candidate fits", () =>
|
||||
Effect.gen(function* () {
|
||||
const photon = yield* Effect.promise(() => import("@silvia-odwyer/photon-node"))
|
||||
|
||||
@@ -393,6 +393,35 @@ You can also configure [local models](/docs/models#local). [Learn more](/docs/mo
|
||||
|
||||
---
|
||||
|
||||
### Image attachments
|
||||
|
||||
OpenCode normalizes image attachments before sending them to the model. By default, images are resized when they exceed `2000x2000` pixels or `5242880` base64 bytes.
|
||||
|
||||
Configure image attachment limits with the `attachment.image` option:
|
||||
|
||||
```json title="opencode.json"
|
||||
{
|
||||
"$schema": "https://opencode.ai/config.json",
|
||||
"attachment": {
|
||||
"image": {
|
||||
"auto_resize": true,
|
||||
"max_width": 2000,
|
||||
"max_height": 2000,
|
||||
"max_base64_bytes": 5242880
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
- `auto_resize` - Resize images that exceed the configured limits before provider requests. Set to `false` to reject oversized images instead.
|
||||
- `max_width` - Maximum image width in pixels before resizing or rejection.
|
||||
- `max_height` - Maximum image height in pixels before resizing or rejection.
|
||||
- `max_base64_bytes` - Maximum encoded image payload size. This is the base64 payload size, not the original file size.
|
||||
|
||||
If an image still cannot fit after resizing, OpenCode omits oversized tool-result images or fails oversized user-provided images with an image size error.
|
||||
|
||||
---
|
||||
|
||||
#### Provider-Specific Options
|
||||
|
||||
Some providers support additional configuration options beyond the generic `timeout` and `apiKey` settings.
|
||||
|
||||
Reference in New Issue
Block a user