diff --git a/bun.lock b/bun.lock index b5971a500c..5b604702c9 100644 --- a/bun.lock +++ b/bun.lock @@ -81,6 +81,8 @@ "@opencode-ai/console-mail": "workspace:*", "@opencode-ai/console-resource": "workspace:*", "@opencode-ai/ui": "workspace:*", + "@smithy/eventstream-codec": "4.2.7", + "@smithy/util-utf8": "4.2.0", "@solidjs/meta": "catalog:", "@solidjs/router": "catalog:", "@solidjs/start": "catalog:", @@ -1522,7 +1524,7 @@ "@smithy/credential-provider-imds": ["@smithy/credential-provider-imds@4.2.5", "", { "dependencies": { "@smithy/node-config-provider": "^4.3.5", "@smithy/property-provider": "^4.2.5", "@smithy/types": "^4.9.0", "@smithy/url-parser": "^4.2.5", "tslib": "^2.6.2" } }, "sha512-BZwotjoZWn9+36nimwm/OLIcVe+KYRwzMjfhd4QT7QxPm9WY0HiOV8t/Wlh+HVUif0SBVV7ksq8//hPaBC/okQ=="], - "@smithy/eventstream-codec": ["@smithy/eventstream-codec@4.2.5", "", { "dependencies": { "@aws-crypto/crc32": "5.2.0", "@smithy/types": "^4.9.0", "@smithy/util-hex-encoding": "^4.2.0", "tslib": "^2.6.2" } }, "sha512-Ogt4Zi9hEbIP17oQMd68qYOHUzmH47UkK7q7Gl55iIm9oKt27MUGrC5JfpMroeHjdkOliOA4Qt3NQ1xMq/nrlA=="], + "@smithy/eventstream-codec": ["@smithy/eventstream-codec@4.2.7", "", { "dependencies": { "@aws-crypto/crc32": "5.2.0", "@smithy/types": "^4.11.0", "@smithy/util-hex-encoding": "^4.2.0", "tslib": "^2.6.2" } }, "sha512-DrpkEoM3j9cBBWhufqBwnbbn+3nf1N9FP6xuVJ+e220jbactKuQgaZwjwP5CP1t+O94brm2JgVMD2atMGX3xIQ=="], "@smithy/eventstream-serde-browser": ["@smithy/eventstream-serde-browser@4.2.5", "", { "dependencies": { "@smithy/eventstream-serde-universal": "^4.2.5", "@smithy/types": "^4.9.0", "tslib": "^2.6.2" } }, "sha512-HohfmCQZjppVnKX2PnXlf47CW3j92Ki6T/vkAT2DhBR47e89pen3s4fIa7otGTtrVxmj7q+IhH0RnC5kpR8wtw=="], @@ -3966,6 +3968,8 @@ "@ai-sdk/amazon-bedrock/@ai-sdk/anthropic": ["@ai-sdk/anthropic@2.0.57", "", { "dependencies": { "@ai-sdk/provider": "2.0.1", "@ai-sdk/provider-utils": "3.0.20" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-DREpYqW2pylgaj69gZ+K8u92bo9DaMgFdictYnY+IwYeY3bawQ4zI7l/o1VkDsBDljAx8iYz5lPURwVZNu+Xpg=="], + "@ai-sdk/amazon-bedrock/@smithy/eventstream-codec": ["@smithy/eventstream-codec@4.2.5", "", { "dependencies": { "@aws-crypto/crc32": "5.2.0", "@smithy/types": "^4.9.0", "@smithy/util-hex-encoding": "^4.2.0", "tslib": "^2.6.2" } }, "sha512-Ogt4Zi9hEbIP17oQMd68qYOHUzmH47UkK7q7Gl55iIm9oKt27MUGrC5JfpMroeHjdkOliOA4Qt3NQ1xMq/nrlA=="], + "@ai-sdk/anthropic/@ai-sdk/provider": ["@ai-sdk/provider@2.0.0", "", { "dependencies": { "json-schema": "^0.4.0" } }, "sha512-6o7Y2SeO9vFKB8lArHXehNuusnpddKPk7xqL7T2/b+OvXMRIXUO1rR4wcv1hAFUAT9avGZshty3Wlua/XA7TvA=="], "@ai-sdk/anthropic/@ai-sdk/provider-utils": ["@ai-sdk/provider-utils@3.0.0", "", { "dependencies": { "@ai-sdk/provider": "2.0.0", "@standard-schema/spec": "^1.0.0", "eventsource-parser": "^3.0.3", "zod-to-json-schema": "^3.24.1" }, "peerDependencies": { "zod": "^3.25.76 || ^4" } }, "sha512-BoQZtGcBxkeSH1zK+SRYNDtJPIPpacTeiMZqnG4Rv6xXjEwM0FH4MGs9c+PlhyEWmQCzjRM2HAotEydFhD4dYw=="], @@ -4236,6 +4240,10 @@ "@slack/web-api/p-queue": ["p-queue@6.6.2", "", { "dependencies": { "eventemitter3": "^4.0.4", "p-timeout": "^3.2.0" } }, "sha512-RwFpb72c/BhQLEXIZ5K2e+AhgNVmIejGlTgiB9MzZ0e93GRvqZ7uSi0dvRF7/XIXDeNkra2fNHBxTyPDGySpjQ=="], + "@smithy/eventstream-codec/@smithy/types": ["@smithy/types@4.11.0", "", { "dependencies": { "tslib": "^2.6.2" } }, "sha512-mlrmL0DRDVe3mNrjTcVcZEgkFmufITfUAPBEA+AHYiIeYyJebso/He1qLbP3PssRe22KUzLRpQSdBPbXdgZ2VA=="], + + "@smithy/eventstream-serde-universal/@smithy/eventstream-codec": ["@smithy/eventstream-codec@4.2.5", "", { "dependencies": { "@aws-crypto/crc32": "5.2.0", "@smithy/types": "^4.9.0", "@smithy/util-hex-encoding": "^4.2.0", "tslib": "^2.6.2" } }, "sha512-Ogt4Zi9hEbIP17oQMd68qYOHUzmH47UkK7q7Gl55iIm9oKt27MUGrC5JfpMroeHjdkOliOA4Qt3NQ1xMq/nrlA=="], + "@solidjs/start/esbuild": ["esbuild@0.25.12", "", { "optionalDependencies": { "@esbuild/aix-ppc64": "0.25.12", "@esbuild/android-arm": "0.25.12", "@esbuild/android-arm64": "0.25.12", "@esbuild/android-x64": "0.25.12", "@esbuild/darwin-arm64": "0.25.12", "@esbuild/darwin-x64": "0.25.12", "@esbuild/freebsd-arm64": "0.25.12", "@esbuild/freebsd-x64": "0.25.12", "@esbuild/linux-arm": "0.25.12", "@esbuild/linux-arm64": "0.25.12", "@esbuild/linux-ia32": "0.25.12", "@esbuild/linux-loong64": "0.25.12", "@esbuild/linux-mips64el": "0.25.12", "@esbuild/linux-ppc64": "0.25.12", "@esbuild/linux-riscv64": "0.25.12", "@esbuild/linux-s390x": "0.25.12", "@esbuild/linux-x64": "0.25.12", "@esbuild/netbsd-arm64": "0.25.12", "@esbuild/netbsd-x64": "0.25.12", "@esbuild/openbsd-arm64": "0.25.12", "@esbuild/openbsd-x64": "0.25.12", "@esbuild/openharmony-arm64": "0.25.12", "@esbuild/sunos-x64": "0.25.12", "@esbuild/win32-arm64": "0.25.12", "@esbuild/win32-ia32": "0.25.12", "@esbuild/win32-x64": "0.25.12" }, "bin": { "esbuild": "bin/esbuild" } }, "sha512-bbPBYYrtZbkt6Os6FiTLCTFxvq4tt3JKall1vRwshA3fdVztsLAatFaZobhkBC8/BrPetoa0oksYoKXoG4ryJg=="], "@solidjs/start/path-to-regexp": ["path-to-regexp@8.3.0", "", {}, "sha512-7jdwVIRtsP8MYpdXSwOS0YdD0Du+qOoF/AEPIt88PcCFrZCzx41oxku1jD88hZBwbNUIEfpqvuhjFaMAqMTWnA=="], diff --git a/packages/console/app/package.json b/packages/console/app/package.json index 75865c4a26..b995a04484 100644 --- a/packages/console/app/package.json +++ b/packages/console/app/package.json @@ -20,6 +20,8 @@ "@opencode-ai/console-mail": "workspace:*", "@opencode-ai/console-resource": "workspace:*", "@opencode-ai/ui": "workspace:*", + "@smithy/eventstream-codec": "4.2.7", + "@smithy/util-utf8": "4.2.0", "@solidjs/meta": "catalog:", "@solidjs/router": "catalog:", "@solidjs/start": "catalog:", diff --git a/packages/console/app/src/routes/zen/util/handler.ts b/packages/console/app/src/routes/zen/util/handler.ts index 2ecc4220a1..0e848886fe 100644 --- a/packages/console/app/src/routes/zen/util/handler.ts +++ b/packages/console/app/src/routes/zen/util/handler.ts @@ -81,12 +81,13 @@ export async function handler( const isTrial = await trialLimiter?.isTrial() const rateLimiter = createRateLimiter(modelInfo.rateLimit, ip) await rateLimiter?.check() - const stickyTracker = createStickyTracker(modelInfo.stickyProvider ?? false, sessionId) + const stickyTracker = createStickyTracker(modelInfo.stickyProvider, sessionId) const stickyProvider = await stickyTracker?.get() const authInfo = await authenticate(modelInfo) const retriableRequest = async (retry: RetryOptions = { excludeProviders: [], retryCount: 0 }) => { const providerInfo = selectProvider( + model, zenData, authInfo, modelInfo, @@ -101,7 +102,7 @@ export async function handler( logger.metric({ provider: providerInfo.id }) const startTimestamp = Date.now() - const reqUrl = providerInfo.modifyUrl(providerInfo.api, providerInfo.model, isStream) + const reqUrl = providerInfo.modifyUrl(providerInfo.api, isStream) const reqBody = JSON.stringify( providerInfo.modifyBody({ ...createBodyConverter(opts.format, providerInfo.format)(body), @@ -135,7 +136,7 @@ export async function handler( // ie. openai 404 error: Item with id 'msg_0ead8b004a3b165d0069436a6b6834819896da85b63b196a3f' not found. res.status !== 404 && // ie. cannot change codex model providers mid-session - !modelInfo.stickyProvider && + modelInfo.stickyProvider !== "strict" && modelInfo.fallbackProvider && providerInfo.id !== modelInfo.fallbackProvider ) { @@ -194,17 +195,19 @@ export async function handler( // Handle streaming response const streamConverter = createStreamPartConverter(providerInfo.format, opts.format) const usageParser = providerInfo.createUsageParser() + const binaryDecoder = providerInfo.createBinaryStreamDecoder() const stream = new ReadableStream({ start(c) { const reader = res.body?.getReader() const decoder = new TextDecoder() const encoder = new TextEncoder() + let buffer = "" let responseLength = 0 function pump(): Promise { return ( - reader?.read().then(async ({ done, value }) => { + reader?.read().then(async ({ done, value: rawValue }) => { if (done) { logger.metric({ response_length: responseLength, @@ -230,6 +233,10 @@ export async function handler( "timestamp.first_byte": now, }) } + + const value = binaryDecoder ? binaryDecoder(rawValue) : rawValue + if (!value) return + responseLength += value.length buffer += decoder.decode(value, { stream: true }) dataDumper?.provideStream(buffer) @@ -331,6 +338,7 @@ export async function handler( } function selectProvider( + reqModel: string, zenData: ZenData, authInfo: AuthInfo, modelInfo: ModelInfo, @@ -339,7 +347,7 @@ export async function handler( retry: RetryOptions, stickyProvider: string | undefined, ) { - const provider = (() => { + const modelProvider = (() => { if (authInfo?.provider?.credentials) { return modelInfo.providers.find((provider) => provider.id === modelInfo.byokProvider) } @@ -372,18 +380,19 @@ export async function handler( return providers[index || 0] })() - if (!provider) throw new ModelError("No provider available") - if (!(provider.id in zenData.providers)) throw new ModelError(`Provider ${provider.id} not supported`) + if (!modelProvider) throw new ModelError("No provider available") + if (!(modelProvider.id in zenData.providers)) throw new ModelError(`Provider ${modelProvider.id} not supported`) return { - ...provider, - ...zenData.providers[provider.id], + ...modelProvider, + ...zenData.providers[modelProvider.id], ...(() => { - const format = zenData.providers[provider.id].format - if (format === "anthropic") return anthropicHelper - if (format === "google") return googleHelper - if (format === "openai") return openaiHelper - return oaCompatHelper + const format = zenData.providers[modelProvider.id].format + const providerModel = modelProvider.model + if (format === "anthropic") return anthropicHelper({ reqModel, providerModel }) + if (format === "google") return googleHelper({ reqModel, providerModel }) + if (format === "openai") return openaiHelper({ reqModel, providerModel }) + return oaCompatHelper({ reqModel, providerModel }) })(), } } diff --git a/packages/console/app/src/routes/zen/util/provider/anthropic.ts b/packages/console/app/src/routes/zen/util/provider/anthropic.ts index 887a6e4b5e..87344c0deb 100644 --- a/packages/console/app/src/routes/zen/util/provider/anthropic.ts +++ b/packages/console/app/src/routes/zen/util/provider/anthropic.ts @@ -1,4 +1,6 @@ +import { EventStreamCodec } from "@smithy/eventstream-codec" import { ProviderHelper, CommonRequest, CommonResponse, CommonChunk } from "./provider" +import { fromUtf8, toUtf8 } from "@smithy/util-utf8" type Usage = { cache_creation?: { @@ -14,65 +16,164 @@ type Usage = { } } -export const anthropicHelper = { - format: "anthropic", - modifyUrl: (providerApi: string) => providerApi + "/messages", - modifyHeaders: (headers: Headers, body: Record, apiKey: string) => { - headers.set("x-api-key", apiKey) - headers.set("anthropic-version", headers.get("anthropic-version") ?? "2023-06-01") - if (body.model.startsWith("claude-sonnet-")) { - headers.set("anthropic-beta", "context-1m-2025-08-07") - } - }, - modifyBody: (body: Record) => { - return { +export const anthropicHelper: ProviderHelper = ({ reqModel, providerModel }) => { + const isBedrockModelArn = providerModel.startsWith("arn:aws:bedrock:") + const isBedrockModelID = providerModel.startsWith("global.anthropic.") + const isBedrock = isBedrockModelArn || isBedrockModelID + const isSonnet = reqModel.includes("sonnet") + return { + format: "anthropic", + modifyUrl: (providerApi: string, isStream?: boolean) => + isBedrock + ? `${providerApi}/model/${isBedrockModelArn ? encodeURIComponent(providerModel) : providerModel}/${isStream ? "invoke-with-response-stream" : "invoke"}` + : providerApi + "/messages", + modifyHeaders: (headers: Headers, body: Record, apiKey: string) => { + if (isBedrock) { + headers.set("Authorization", `Bearer ${apiKey}`) + } else { + headers.set("x-api-key", apiKey) + headers.set("anthropic-version", headers.get("anthropic-version") ?? "2023-06-01") + if (body.model.startsWith("claude-sonnet-")) { + headers.set("anthropic-beta", "context-1m-2025-08-07") + } + } + }, + modifyBody: (body: Record) => ({ ...body, - service_tier: "standard_only", - } - }, - streamSeparator: "\n\n", - createUsageParser: () => { - let usage: Usage + ...(isBedrock + ? { + anthropic_version: "bedrock-2023-05-31", + anthropic_beta: isSonnet ? "context-1m-2025-08-07" : undefined, + model: undefined, + stream: undefined, + } + : { + service_tier: "standard_only", + }), + }), + createBinaryStreamDecoder: () => { + if (!isBedrock) return undefined - return { - parse: (chunk: string) => { - const data = chunk.split("\n")[1] - if (!data.startsWith("data: ")) return + const decoder = new TextDecoder() + const encoder = new TextEncoder() + const codec = new EventStreamCodec(toUtf8, fromUtf8) + let buffer = new Uint8Array(0) + return (value: Uint8Array) => { + const newBuffer = new Uint8Array(buffer.length + value.length) + newBuffer.set(buffer) + newBuffer.set(value, buffer.length) + buffer = newBuffer + + if (buffer.length < 4) return + // The first 4 bytes are the total length (big-endian). + const totalLength = new DataView(buffer.buffer, buffer.byteOffset, buffer.byteLength).getUint32(0, false) + + // If we don't have the full message yet, wait for more chunks. + if (buffer.length < totalLength) return - let json try { - json = JSON.parse(data.slice(6)) - } catch (e) { - return - } + // Decode exactly the sub-slice for this event. + const subView = buffer.subarray(0, totalLength) + const decoded = codec.decode(subView) - const usageUpdate = json.usage ?? json.message?.usage - if (!usageUpdate) return - usage = { - ...usage, - ...usageUpdate, - cache_creation: { - ...usage?.cache_creation, - ...usageUpdate.cache_creation, - }, - server_tool_use: { - ...usage?.server_tool_use, - ...usageUpdate.server_tool_use, - }, + // Slice the used bytes out of the buffer, removing this message. + buffer = buffer.slice(totalLength) + + // Process message + /* Example of Bedrock data + ``` + { + bytes: 'eyJ0eXBlIjoibWVzc2FnZV9zdGFydCIsIm1lc3NhZ2UiOnsibW9kZWwiOiJjbGF1ZGUtb3B1cy00LTUtMjAyNTExMDEiLCJpZCI6Im1zZ19iZHJrXzAxMjVGdHRGb2lkNGlwWmZ4SzZMbktxeCIsInR5cGUiOiJtZXNzYWdlIiwicm9sZSI6ImFzc2lzdGFudCIsImNvbnRlbnQiOltdLCJzdG9wX3JlYXNvbiI6bnVsbCwic3RvcF9zZXF1ZW5jZSI6bnVsbCwidXNhZ2UiOnsiaW5wdXRfdG9rZW5zIjo0LCJjYWNoZV9jcmVhdGlvbl9pbnB1dF90b2tlbnMiOjEsImNhY2hlX3JlYWRfaW5wdXRfdG9rZW5zIjoxMTk2MywiY2FjaGVfY3JlYXRpb24iOnsiZXBoZW1lcmFsXzVtX2lucHV0X3Rva2VucyI6MSwiZXBoZW1lcmFsXzFoX2lucHV0X3Rva2VucyI6MH0sIm91dHB1dF90b2tlbnMiOjF9fX0=', + p: '...' } - }, - retrieve: () => usage, - } - }, - normalizeUsage: (usage: Usage) => ({ - inputTokens: usage.input_tokens ?? 0, - outputTokens: usage.output_tokens ?? 0, - reasoningTokens: undefined, - cacheReadTokens: usage.cache_read_input_tokens ?? undefined, - cacheWrite5mTokens: usage.cache_creation?.ephemeral_5m_input_tokens ?? undefined, - cacheWrite1hTokens: usage.cache_creation?.ephemeral_1h_input_tokens ?? undefined, - }), -} satisfies ProviderHelper + ``` + + Decoded bytes + ``` + { + type: 'message_start', + message: { + model: 'claude-opus-4-5-20251101', + id: 'msg_bdrk_0125FttFoid4ipZfxK6LnKqx', + type: 'message', + role: 'assistant', + content: [], + stop_reason: null, + stop_sequence: null, + usage: { + input_tokens: 4, + cache_creation_input_tokens: 1, + cache_read_input_tokens: 11963, + cache_creation: [Object], + output_tokens: 1 + } + } + } + ``` + */ + + /* Example of Anthropic data + ``` + event: message_delta + data: {"type":"message_start","message":{"model":"claude-opus-4-5-20251101","id":"msg_01ETvwVWSKULxzPdkQ1xAnk2","type":"message","role":"assistant","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":3,"cache_creation_input_tokens":11543,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":11543,"ephemeral_1h_input_tokens":0},"output_tokens":1,"service_tier":"standard"}}} + ``` + */ + if (decoded.headers[":message-type"]?.value !== "event") return + const data = decoder.decode(decoded.body, { stream: true }) + + const parsedDataResult = JSON.parse(data) + delete parsedDataResult.p + const utf8 = atob(parsedDataResult.bytes) + return encoder.encode(["event: message_start", "\n", "data: " + utf8, "\n\n"].join("")) + } catch (e) { + console.log(e) + } + } + }, + streamSeparator: "\n\n", + createUsageParser: () => { + let usage: Usage + + return { + parse: (chunk: string) => { + const data = chunk.split("\n")[1] + if (!data.startsWith("data: ")) return + + let json + try { + json = JSON.parse(data.slice(6)) + } catch (e) { + return + } + + const usageUpdate = json.usage ?? json.message?.usage + if (!usageUpdate) return + usage = { + ...usage, + ...usageUpdate, + cache_creation: { + ...usage?.cache_creation, + ...usageUpdate.cache_creation, + }, + server_tool_use: { + ...usage?.server_tool_use, + ...usageUpdate.server_tool_use, + }, + } + }, + retrieve: () => usage, + } + }, + normalizeUsage: (usage: Usage) => ({ + inputTokens: usage.input_tokens ?? 0, + outputTokens: usage.output_tokens ?? 0, + reasoningTokens: undefined, + cacheReadTokens: usage.cache_read_input_tokens ?? undefined, + cacheWrite5mTokens: usage.cache_creation?.ephemeral_5m_input_tokens ?? undefined, + cacheWrite1hTokens: usage.cache_creation?.ephemeral_1h_input_tokens ?? undefined, + }), + } +} export function fromAnthropicRequest(body: any): CommonRequest { if (!body || typeof body !== "object") return body diff --git a/packages/console/app/src/routes/zen/util/provider/google.ts b/packages/console/app/src/routes/zen/util/provider/google.ts index afde42096c..f6f7d6e19b 100644 --- a/packages/console/app/src/routes/zen/util/provider/google.ts +++ b/packages/console/app/src/routes/zen/util/provider/google.ts @@ -26,16 +26,17 @@ type Usage = { thoughtsTokenCount?: number } -export const googleHelper = { +export const googleHelper: ProviderHelper = ({ providerModel }) => ({ format: "google", - modifyUrl: (providerApi: string, model?: string, isStream?: boolean) => - `${providerApi}/models/${model}:${isStream ? "streamGenerateContent?alt=sse" : "generateContent"}`, + modifyUrl: (providerApi: string, isStream?: boolean) => + `${providerApi}/models/${providerModel}:${isStream ? "streamGenerateContent?alt=sse" : "generateContent"}`, modifyHeaders: (headers: Headers, body: Record, apiKey: string) => { headers.set("x-goog-api-key", apiKey) }, modifyBody: (body: Record) => { return body }, + createBinaryStreamDecoder: () => undefined, streamSeparator: "\r\n\r\n", createUsageParser: () => { let usage: Usage @@ -71,4 +72,4 @@ export const googleHelper = { cacheWrite1hTokens: undefined, } }, -} satisfies ProviderHelper +}) diff --git a/packages/console/app/src/routes/zen/util/provider/openai-compatible.ts b/packages/console/app/src/routes/zen/util/provider/openai-compatible.ts index 5771ed4faa..699243d085 100644 --- a/packages/console/app/src/routes/zen/util/provider/openai-compatible.ts +++ b/packages/console/app/src/routes/zen/util/provider/openai-compatible.ts @@ -21,7 +21,7 @@ type Usage = { } } -export const oaCompatHelper = { +export const oaCompatHelper: ProviderHelper = () => ({ format: "oa-compat", modifyUrl: (providerApi: string) => providerApi + "/chat/completions", modifyHeaders: (headers: Headers, body: Record, apiKey: string) => { @@ -33,6 +33,7 @@ export const oaCompatHelper = { ...(body.stream ? { stream_options: { include_usage: true } } : {}), } }, + createBinaryStreamDecoder: () => undefined, streamSeparator: "\n\n", createUsageParser: () => { let usage: Usage @@ -68,7 +69,7 @@ export const oaCompatHelper = { cacheWrite1hTokens: undefined, } }, -} satisfies ProviderHelper +}) export function fromOaCompatibleRequest(body: any): CommonRequest { if (!body || typeof body !== "object") return body diff --git a/packages/console/app/src/routes/zen/util/provider/openai.ts b/packages/console/app/src/routes/zen/util/provider/openai.ts index dff6e13fbe..f4d7699e97 100644 --- a/packages/console/app/src/routes/zen/util/provider/openai.ts +++ b/packages/console/app/src/routes/zen/util/provider/openai.ts @@ -12,7 +12,7 @@ type Usage = { total_tokens?: number } -export const openaiHelper = { +export const openaiHelper: ProviderHelper = () => ({ format: "openai", modifyUrl: (providerApi: string) => providerApi + "/responses", modifyHeaders: (headers: Headers, body: Record, apiKey: string) => { @@ -21,6 +21,7 @@ export const openaiHelper = { modifyBody: (body: Record) => { return body }, + createBinaryStreamDecoder: () => undefined, streamSeparator: "\n\n", createUsageParser: () => { let usage: Usage @@ -58,7 +59,7 @@ export const openaiHelper = { cacheWrite1hTokens: undefined, } }, -} satisfies ProviderHelper +}) export function fromOpenaiRequest(body: any): CommonRequest { if (!body || typeof body !== "object") return body diff --git a/packages/console/app/src/routes/zen/util/provider/provider.ts b/packages/console/app/src/routes/zen/util/provider/provider.ts index 730ad5a278..bbf54f4f96 100644 --- a/packages/console/app/src/routes/zen/util/provider/provider.ts +++ b/packages/console/app/src/routes/zen/util/provider/provider.ts @@ -33,11 +33,12 @@ export type UsageInfo = { cacheWrite1hTokens?: number } -export type ProviderHelper = { +export type ProviderHelper = (input: { reqModel: string; providerModel: string }) => { format: ZenData.Format - modifyUrl: (providerApi: string, model?: string, isStream?: boolean) => string + modifyUrl: (providerApi: string, isStream?: boolean) => string modifyHeaders: (headers: Headers, body: Record, apiKey: string) => void modifyBody: (body: Record) => Record + createBinaryStreamDecoder: () => ((chunk: Uint8Array) => Uint8Array | undefined) | undefined streamSeparator: string createUsageParser: () => { parse: (chunk: string) => void diff --git a/packages/console/app/src/routes/zen/util/stickyProviderTracker.ts b/packages/console/app/src/routes/zen/util/stickyProviderTracker.ts index 63cbb0a68c..8029757c5b 100644 --- a/packages/console/app/src/routes/zen/util/stickyProviderTracker.ts +++ b/packages/console/app/src/routes/zen/util/stickyProviderTracker.ts @@ -1,6 +1,6 @@ import { Resource } from "@opencode-ai/console-resource" -export function createStickyTracker(stickyProvider: boolean, session: string) { +export function createStickyTracker(stickyProvider: "strict" | "prefer" | undefined, session: string) { if (!stickyProvider) return if (!session) return const key = `sticky:${session}` diff --git a/packages/console/core/src/model.ts b/packages/console/core/src/model.ts index 2c150c7c4e..0fd8bdecfb 100644 --- a/packages/console/core/src/model.ts +++ b/packages/console/core/src/model.ts @@ -35,7 +35,7 @@ export namespace ZenData { cost200K: ModelCostSchema.optional(), allowAnonymous: z.boolean().optional(), byokProvider: z.enum(["openai", "anthropic", "google"]).optional(), - stickyProvider: z.boolean().optional(), + stickyProvider: z.enum(["strict", "prefer"]).optional(), trial: TrialSchema.optional(), rateLimit: z.number().optional(), fallbackProvider: z.string().optional(), diff --git a/packages/console/core/sst-env.d.ts b/packages/console/core/sst-env.d.ts index b8e50a2611..3710cb77f8 100644 --- a/packages/console/core/sst-env.d.ts +++ b/packages/console/core/sst-env.d.ts @@ -134,6 +134,10 @@ declare module "sst" { "type": "sst.sst.Secret" "value": string } + "ZEN_MODELS8": { + "type": "sst.sst.Secret" + "value": string + } "ZEN_SESSION_SECRET": { "type": "sst.sst.Secret" "value": string diff --git a/packages/console/function/sst-env.d.ts b/packages/console/function/sst-env.d.ts index b8e50a2611..3710cb77f8 100644 --- a/packages/console/function/sst-env.d.ts +++ b/packages/console/function/sst-env.d.ts @@ -134,6 +134,10 @@ declare module "sst" { "type": "sst.sst.Secret" "value": string } + "ZEN_MODELS8": { + "type": "sst.sst.Secret" + "value": string + } "ZEN_SESSION_SECRET": { "type": "sst.sst.Secret" "value": string diff --git a/packages/console/resource/sst-env.d.ts b/packages/console/resource/sst-env.d.ts index b8e50a2611..3710cb77f8 100644 --- a/packages/console/resource/sst-env.d.ts +++ b/packages/console/resource/sst-env.d.ts @@ -134,6 +134,10 @@ declare module "sst" { "type": "sst.sst.Secret" "value": string } + "ZEN_MODELS8": { + "type": "sst.sst.Secret" + "value": string + } "ZEN_SESSION_SECRET": { "type": "sst.sst.Secret" "value": string diff --git a/packages/enterprise/sst-env.d.ts b/packages/enterprise/sst-env.d.ts index b8e50a2611..3710cb77f8 100644 --- a/packages/enterprise/sst-env.d.ts +++ b/packages/enterprise/sst-env.d.ts @@ -134,6 +134,10 @@ declare module "sst" { "type": "sst.sst.Secret" "value": string } + "ZEN_MODELS8": { + "type": "sst.sst.Secret" + "value": string + } "ZEN_SESSION_SECRET": { "type": "sst.sst.Secret" "value": string diff --git a/packages/function/sst-env.d.ts b/packages/function/sst-env.d.ts index b8e50a2611..3710cb77f8 100644 --- a/packages/function/sst-env.d.ts +++ b/packages/function/sst-env.d.ts @@ -134,6 +134,10 @@ declare module "sst" { "type": "sst.sst.Secret" "value": string } + "ZEN_MODELS8": { + "type": "sst.sst.Secret" + "value": string + } "ZEN_SESSION_SECRET": { "type": "sst.sst.Secret" "value": string diff --git a/sst-env.d.ts b/sst-env.d.ts index 3160fc165b..ad02b7bf1d 100644 --- a/sst-env.d.ts +++ b/sst-env.d.ts @@ -160,6 +160,10 @@ declare module "sst" { "type": "sst.sst.Secret" "value": string } + "ZEN_MODELS8": { + "type": "sst.sst.Secret" + "value": string + } "ZEN_SESSION_SECRET": { "type": "sst.sst.Secret" "value": string