diff --git a/packages/llm/src/schema/errors.ts b/packages/llm/src/schema/errors.ts index 9bcc8e1694..39bf5b6252 100644 --- a/packages/llm/src/schema/errors.ts +++ b/packages/llm/src/schema/errors.ts @@ -198,5 +198,6 @@ export class LLMError extends Schema.TaggedErrorClass()("LLM.Error", { */ export class ToolFailure extends Schema.TaggedErrorClass()("LLM.ToolFailure", { message: Schema.String, + error: Schema.optional(Schema.Defect), metadata: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)), }) {} diff --git a/packages/llm/src/schema/events.ts b/packages/llm/src/schema/events.ts index 6a088dc873..63c9b7b7df 100644 --- a/packages/llm/src/schema/events.ts +++ b/packages/llm/src/schema/events.ts @@ -171,6 +171,7 @@ export const ToolError = Schema.Struct({ id: ToolCallID, name: Schema.String, message: Schema.String, + error: Schema.optional(Schema.Defect), providerMetadata: Schema.optional(ProviderMetadata), }).annotate({ identifier: "LLM.Event.ToolError" }) export type ToolError = Schema.Schema.Type diff --git a/packages/llm/src/tool-runtime.ts b/packages/llm/src/tool-runtime.ts index d83dcc67ad..ef527faa21 100644 --- a/packages/llm/src/tool-runtime.ts +++ b/packages/llm/src/tool-runtime.ts @@ -112,17 +112,29 @@ export const stream = (options: StreamOptions): Stream.Strea const dispatched = yield* Effect.forEach( state.toolCalls, - (call) => dispatch(tools, call).pipe(Effect.map((result) => [call, result] as const)), + (call) => + dispatch(tools, call).pipe(Effect.map((result) => [call, result.result, result.error] as const)), { concurrency }, ) - const resultStream = Stream.fromIterable(dispatched.flatMap(([call, result]) => emitEvents(call, result))) + const resultStream = Stream.fromIterable( + dispatched.flatMap(([call, result, error]) => emitEvents(call, result, error)), + ) if (!options.stopWhen) return resultStream.pipe(Stream.concat(finishStream)) if (options.stopWhen({ step, request })) return resultStream.pipe(Stream.concat(finishStream)) return resultStream.pipe( Stream.concat( - loop(followUpRequest(request, state, dispatched), step + 1, totalUsage, totalProviderMetadata), + loop( + followUpRequest( + request, + state, + dispatched.map(([call, result]) => [call, result] as const), + ), + step + 1, + totalUsage, + totalProviderMetadata, + ), ), ) }), @@ -215,7 +227,7 @@ const addUsage = (left: Usage | undefined, right: Usage | undefined) => { | "reasoningTokens" | "totalTokens" const sum = (key: UsageKey) => - left[key] === undefined && right[key] === undefined ? undefined : Number(left[key] ?? 0) + Number(right[key] ?? 0) + left[key] === undefined && right[key] === undefined ? undefined : (left[key] ?? 0) + (right[key] ?? 0) return new Usage({ inputTokens: sum("inputTokens"), @@ -264,16 +276,20 @@ const appendStreamingText = ( state.assistantContent.push({ type, text, providerMetadata }) } -const dispatch = (tools: Tools, call: ToolCallPart): Effect.Effect => { +const dispatch = (tools: Tools, call: ToolCallPart): Effect.Effect<{ result: ToolResultValue; error?: unknown }> => { const tool = tools[call.name] - if (!tool) return Effect.succeed({ type: "error" as const, value: `Unknown tool: ${call.name}` }) + if (!tool) return Effect.succeed({ result: { type: "error" as const, value: `Unknown tool: ${call.name}` } }) if (!tool.execute) - return Effect.succeed({ type: "error" as const, value: `Tool has no execute handler: ${call.name}` }) + return Effect.succeed({ result: { type: "error" as const, value: `Tool has no execute handler: ${call.name}` } }) return decodeAndExecute(tool, call).pipe( Effect.catchTag("LLM.ToolFailure", (failure) => - Effect.succeed({ type: "error" as const, value: failure.message } satisfies ToolResultValue), + Effect.succeed({ + result: { type: "error" as const, value: failure.message } satisfies ToolResultValue, + error: failure.error, + }), ), + Effect.map((result) => ("result" in result ? result : { result })), ) } @@ -294,10 +310,10 @@ const decodeAndExecute = (tool: AnyTool, call: ToolCallPart): Effect.Effect ({ type: "json", value: encoded })), ) -const emitEvents = (call: ToolCallPart, result: ToolResultValue): ReadonlyArray => +const emitEvents = (call: ToolCallPart, result: ToolResultValue, error: unknown): ReadonlyArray => result.type === "error" ? [ - LLMEvent.toolError({ id: call.id, name: call.name, message: String(result.value) }), + LLMEvent.toolError({ id: call.id, name: call.name, message: String(result.value), error }), LLMEvent.toolResult({ id: call.id, name: call.name, result }), ] : [LLMEvent.toolResult({ id: call.id, name: call.name, result })] diff --git a/packages/llm/test/tool-runtime.test.ts b/packages/llm/test/tool-runtime.test.ts index 573021c4c2..81389a466b 100644 --- a/packages/llm/test/tool-runtime.test.ts +++ b/packages/llm/test/tool-runtime.test.ts @@ -25,6 +25,7 @@ const baseRequest = LLM.request({ model, prompt: "Use the tool.", }) +const weatherFailureCause = new Error("weather lookup denied") const get_weather = tool({ description: "Get current weather for a city.", @@ -32,7 +33,8 @@ const get_weather = tool({ success: Schema.Struct({ temperature: Schema.Number, condition: Schema.String }), execute: ({ city }) => Effect.gen(function* () { - if (city === "FAIL") return yield* new ToolFailure({ message: `Weather lookup failed for ${city}` }) + if (city === "FAIL") + return yield* new ToolFailure({ message: `Weather lookup failed for ${city}`, error: weatherFailureCause }) return { temperature: 22, condition: "sunny" } }), }) @@ -85,23 +87,27 @@ describe("LLMClient tools", () => { tools: { get_weather }, }).pipe(Stream.runCollect, Effect.provide(layer)) - const second = bodies[1] as { - readonly messages?: ReadonlyArray> - readonly tools?: ReadonlyArray - readonly tool_choice?: unknown - readonly max_tokens?: unknown - } + const second = bodies[1] + if (!second || typeof second !== "object") throw new Error("Expected second request body") + const messages = Reflect.get(second, "messages") + const tools = Reflect.get(second, "tools") - expect(second.max_tokens).toBe(50) - expect(second.tool_choice).toBe("auto") - expect(second.tools).toHaveLength(1) - expect(second.messages?.map((message) => message.role)).toEqual(["user", "assistant", "tool"]) - expect(second.messages?.[1]).toMatchObject({ + expect(Reflect.get(second, "max_tokens")).toBe(50) + expect(Reflect.get(second, "tool_choice")).toBe("auto") + expect(tools).toHaveLength(1) + expect( + Array.isArray(messages) + ? messages.map((message) => + message && typeof message === "object" ? Reflect.get(message, "role") : undefined, + ) + : undefined, + ).toEqual(["user", "assistant", "tool"]) + expect(Array.isArray(messages) ? messages[1] : undefined).toMatchObject({ role: "assistant", content: null, tool_calls: [{ id: "call_1", type: "function", function: { name: "get_weather" } }], }) - expect(second.messages?.[2]).toMatchObject({ + expect(Array.isArray(messages) ? messages[2] : undefined).toMatchObject({ role: "tool", tool_call_id: "call_1", content: '{"temperature":22,"condition":"sunny"}', @@ -327,6 +333,7 @@ describe("LLMClient tools", () => { const toolError = events.find(LLMEvent.is.toolError) expect(toolError).toMatchObject({ type: "tool-error", id: "call_1", name: "get_weather" }) expect(toolError?.message).toBe("Weather lookup failed for FAIL") + expect(toolError?.error).toBe(weatherFailureCause) }), )