Compare commits

...

2 Commits

Author SHA1 Message Date
James Long
f5dde52cc4 Fix types 2026-03-06 14:22:10 -05:00
James Long
5df0c6e3c3 feat(core): add chunkTimeout option for sse timeouts in providers 2026-03-06 14:06:41 -05:00
4 changed files with 77 additions and 10 deletions

View File

@@ -972,6 +972,14 @@ export namespace Config {
.describe(
"Timeout in milliseconds for requests to this provider. Default is 300000 (5 minutes). Set to false to disable timeout.",
),
chunkTimeout: z
.number()
.int()
.positive()
.optional()
.describe(
"Timeout in milliseconds between streamed SSE chunks for this provider. If no chunk arrives within this window, the request is aborted.",
),
})
.catchall(z.any())
.optional(),

View File

@@ -46,6 +46,8 @@ import { GoogleAuth } from "google-auth-library"
import { ProviderTransform } from "./transform"
import { Installation } from "../installation"
const DEFAULT_CHUNK_TIMEOUT = 120_000
export namespace Provider {
const log = Log.create({ service: "provider" })
@@ -85,6 +87,54 @@ export namespace Provider {
})
}
function wrapSSE(res: Response, ms: number, ctl: AbortController) {
if (typeof ms !== "number" || ms <= 0) return res
if (!res.body) return res
if (!res.headers.get("content-type")?.includes("text/event-stream")) return res
const reader = res.body.getReader()
const body = new ReadableStream<Uint8Array>({
async pull(ctrl) {
const part = await new Promise<Awaited<ReturnType<typeof reader.read>>>((resolve, reject) => {
const id = setTimeout(() => {
const err = new Error("SSE read timed out")
ctl.abort(err)
void reader.cancel(err)
reject(err)
}, ms)
reader.read().then(
(part) => {
clearTimeout(id)
resolve(part)
},
(err) => {
clearTimeout(id)
reject(err)
},
)
})
if (part.done) {
ctrl.close()
return
}
ctrl.enqueue(part.value)
},
async cancel(reason) {
ctl.abort(reason)
await reader.cancel(reason)
},
})
return new Response(body, {
headers: new Headers(res.headers),
status: res.status,
statusText: res.statusText,
})
}
const BUNDLED_PROVIDERS: Record<string, (options: any) => SDK> = {
"@ai-sdk/amazon-bedrock": createAmazonBedrock,
"@ai-sdk/anthropic": createAnthropic,
@@ -1091,21 +1141,23 @@ export namespace Provider {
if (existing) return existing
const customFetch = options["fetch"]
const chunkTimeout = options["chunkTimeout"] || DEFAULT_CHUNK_TIMEOUT
delete options["chunkTimeout"]
options["fetch"] = async (input: any, init?: BunFetchRequestInit) => {
// Preserve custom fetch if it exists, wrap it with timeout logic
const fetchFn = customFetch ?? fetch
const opts = init ?? {}
const chunkAbortCtl = typeof chunkTimeout === "number" && chunkTimeout > 0 ? new AbortController() : undefined
const signals: AbortSignal[] = []
if (options["timeout"] !== undefined && options["timeout"] !== null) {
const signals: AbortSignal[] = []
if (opts.signal) signals.push(opts.signal)
if (options["timeout"] !== false) signals.push(AbortSignal.timeout(options["timeout"]))
if (opts.signal) signals.push(opts.signal)
if (chunkAbortCtl) signals.push(chunkAbortCtl.signal)
if (options["timeout"] !== undefined && options["timeout"] !== null && options["timeout"] !== false)
signals.push(AbortSignal.timeout(options["timeout"]))
const combined = signals.length > 1 ? AbortSignal.any(signals) : signals[0]
opts.signal = combined
}
const combined = signals.length === 0 ? null : signals.length === 1 ? signals[0] : AbortSignal.any(signals)
if (combined) opts.signal = combined
// Strip openai itemId metadata following what codex does
// Codex uses #[serde(skip_serializing)] on id fields for all item types:
@@ -1125,11 +1177,14 @@ export namespace Provider {
}
}
return fetchFn(input, {
const res = await fetchFn(input, {
...opts,
// @ts-ignore see here: https://github.com/oven-sh/bun/issues/16682
timeout: false,
})
if (!chunkAbortCtl) return res
return wrapSSE(res, chunkTimeout, chunkAbortCtl)
}
const bundledFn = BUNDLED_PROVIDERS[model.api.npm]

View File

@@ -260,6 +260,7 @@ test("env variable takes precedence, config merges options", async () => {
anthropic: {
options: {
timeout: 60000,
chunkTimeout: 15000,
},
},
},
@@ -277,6 +278,7 @@ test("env variable takes precedence, config merges options", async () => {
expect(providers["anthropic"]).toBeDefined()
// Config options should be merged
expect(providers["anthropic"].options.timeout).toBe(60000)
expect(providers["anthropic"].options.chunkTimeout).toBe(15000)
},
})
})

View File

@@ -244,7 +244,7 @@ You can configure the providers and models you want to use in your OpenCode conf
The `small_model` option configures a separate model for lightweight tasks like title generation. By default, OpenCode tries to use a cheaper model if one is available from your provider, otherwise it falls back to your main model.
Provider options can include `timeout` and `setCacheKey`:
Provider options can include `timeout`, `chunkTimeout`, and `setCacheKey`:
```json title="opencode.json"
{
@@ -253,6 +253,7 @@ Provider options can include `timeout` and `setCacheKey`:
"anthropic": {
"options": {
"timeout": 600000,
"chunkTimeout": 30000,
"setCacheKey": true
}
}
@@ -261,6 +262,7 @@ Provider options can include `timeout` and `setCacheKey`:
```
- `timeout` - Request timeout in milliseconds (default: 300000). Set to `false` to disable.
- `chunkTimeout` - Timeout in milliseconds between streamed response chunks. If no chunk arrives in time, the request is aborted.
- `setCacheKey` - Ensure a cache key is always set for designated provider.
You can also configure [local models](/docs/models#local). [Learn more](/docs/models).