test: add retry behavior tests for structured output

Add 5 new unit tests that verify the retry mechanism for structured output validation: - Multiple validation failures trigger multiple onError calls - Success after failures correctly calls onSuccess - Error messages guide model to fix issues and retry - Simulates retry state tracking matching prompt.ts logic - Simulates successful retry after initial failures Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
2026-04-24 14:55:19 +00:00 · 2026-01-19 11:28:29 -08:00
parent 4c7c65a054
commit b1da5714d7
1 changed files with 447 additions and 1 deletions
--- a/packages/opencode/test/session/structured-output.test.ts
+++ b/packages/opencode/test/session/structured-output.test.ts
@@ -160,6 +160,7 @@ describe("structured-output.createStructuredOutputTool", () => {
    const tool = SessionPrompt.createStructuredOutputTool({
      schema: { type: "object", properties: { name: { type: "string" } } },
      onSuccess: () => {},
+      onError: () => {},
    })

    // AI SDK tool type doesn't expose id, but we set it internally
@@ -170,6 +171,7 @@ describe("structured-output.createStructuredOutputTool", () => {
    const tool = SessionPrompt.createStructuredOutputTool({
      schema: { type: "object" },
      onSuccess: () => {},
+      onError: () => {},
    })

    expect(tool.description).toContain("structured format")
@@ -188,6 +190,7 @@ describe("structured-output.createStructuredOutputTool", () => {
    const tool = SessionPrompt.createStructuredOutputTool({
      schema,
      onSuccess: () => {},
+      onError: () => {},
    })

    // AI SDK wraps schema in { jsonSchema: {...} }
@@ -207,6 +210,7 @@ describe("structured-output.createStructuredOutputTool", () => {
    const tool = SessionPrompt.createStructuredOutputTool({
      schema,
      onSuccess: () => {},
+      onError: () => {},
    })

    // AI SDK wraps schema in { jsonSchema: {...} }
@@ -214,7 +218,7 @@ describe("structured-output.createStructuredOutputTool", () => {
    expect(inputSchema.jsonSchema?.$schema).toBeUndefined()
  })

-  test("execute calls onSuccess with args", async () => {
+  test("execute calls onSuccess with valid args", async () => {
    let capturedOutput: unknown

    const tool = SessionPrompt.createStructuredOutputTool({
@@ -222,6 +226,7 @@ describe("structured-output.createStructuredOutputTool", () => {
      onSuccess: (output) => {
        capturedOutput = output
      },
+      onError: () => {},
    })

    expect(tool.execute).toBeDefined()
@@ -237,10 +242,217 @@ describe("structured-output.createStructuredOutputTool", () => {
    expect(result.metadata.valid).toBe(true)
  })

+  test("execute calls onError when validation fails - missing required field", async () => {
+    let capturedError: string | undefined
+    let successCalled = false
+
+    const tool = SessionPrompt.createStructuredOutputTool({
+      schema: {
+        type: "object",
+        properties: {
+          name: { type: "string" },
+          age: { type: "number" },
+        },
+        required: ["name", "age"],
+      },
+      onSuccess: () => {
+        successCalled = true
+      },
+      onError: (error) => {
+        capturedError = error
+      },
+    })
+
+    // Missing required 'age' field
+    const result = await tool.execute!({ name: "Test" }, {
+      toolCallId: "test-call-id",
+      messages: [],
+      abortSignal: undefined as any,
+    })
+
+    expect(successCalled).toBe(false)
+    expect(capturedError).toBeDefined()
+    expect(capturedError).toContain("age")
+    expect(result.output).toContain("Validation failed")
+    expect(result.metadata.valid).toBe(false)
+    expect(result.metadata.error).toBeDefined()
+  })
+
+  test("execute calls onError when validation fails - wrong type", async () => {
+    let capturedError: string | undefined
+    let successCalled = false
+
+    const tool = SessionPrompt.createStructuredOutputTool({
+      schema: {
+        type: "object",
+        properties: {
+          count: { type: "number" },
+        },
+        required: ["count"],
+      },
+      onSuccess: () => {
+        successCalled = true
+      },
+      onError: (error) => {
+        capturedError = error
+      },
+    })
+
+    // Wrong type - string instead of number
+    const result = await tool.execute!({ count: "not a number" }, {
+      toolCallId: "test-call-id",
+      messages: [],
+      abortSignal: undefined as any,
+    })
+
+    expect(successCalled).toBe(false)
+    expect(capturedError).toBeDefined()
+    expect(result.output).toContain("Validation failed")
+    expect(result.metadata.valid).toBe(false)
+  })
+
+  test("execute validates nested objects", async () => {
+    let capturedOutput: unknown
+    let capturedError: string | undefined
+
+    const tool = SessionPrompt.createStructuredOutputTool({
+      schema: {
+        type: "object",
+        properties: {
+          user: {
+            type: "object",
+            properties: {
+              name: { type: "string" },
+              email: { type: "string" },
+            },
+            required: ["name"],
+          },
+        },
+        required: ["user"],
+      },
+      onSuccess: (output) => {
+        capturedOutput = output
+      },
+      onError: (error) => {
+        capturedError = error
+      },
+    })
+
+    // Valid nested object
+    const validResult = await tool.execute!({ user: { name: "John", email: "john@test.com" } }, {
+      toolCallId: "test-call-id",
+      messages: [],
+      abortSignal: undefined as any,
+    })
+
+    expect(capturedOutput).toEqual({ user: { name: "John", email: "john@test.com" } })
+    expect(validResult.metadata.valid).toBe(true)
+
+    // Invalid nested object - missing required 'name'
+    capturedOutput = undefined
+    const invalidResult = await tool.execute!({ user: { email: "john@test.com" } }, {
+      toolCallId: "test-call-id",
+      messages: [],
+      abortSignal: undefined as any,
+    })
+
+    expect(capturedOutput).toBeUndefined()
+    expect(capturedError).toBeDefined()
+    expect(invalidResult.metadata.valid).toBe(false)
+  })
+
+  test("execute validates arrays", async () => {
+    let capturedOutput: unknown
+    let capturedError: string | undefined
+
+    const tool = SessionPrompt.createStructuredOutputTool({
+      schema: {
+        type: "object",
+        properties: {
+          tags: {
+            type: "array",
+            items: { type: "string" },
+          },
+        },
+        required: ["tags"],
+      },
+      onSuccess: (output) => {
+        capturedOutput = output
+      },
+      onError: (error) => {
+        capturedError = error
+      },
+    })
+
+    // Valid array
+    const validResult = await tool.execute!({ tags: ["a", "b", "c"] }, {
+      toolCallId: "test-call-id",
+      messages: [],
+      abortSignal: undefined as any,
+    })
+
+    expect(capturedOutput).toEqual({ tags: ["a", "b", "c"] })
+    expect(validResult.metadata.valid).toBe(true)
+
+    // Invalid array - contains non-string
+    capturedOutput = undefined
+    const invalidResult = await tool.execute!({ tags: ["a", 123, "c"] }, {
+      toolCallId: "test-call-id",
+      messages: [],
+      abortSignal: undefined as any,
+    })
+
+    expect(capturedOutput).toBeUndefined()
+    expect(capturedError).toBeDefined()
+    expect(invalidResult.metadata.valid).toBe(false)
+  })
+
+  test("error message includes path for nested validation errors", async () => {
+    let capturedError: string | undefined
+
+    const tool = SessionPrompt.createStructuredOutputTool({
+      schema: {
+        type: "object",
+        properties: {
+          company: {
+            type: "object",
+            properties: {
+              details: {
+                type: "object",
+                properties: {
+                  foundedYear: { type: "number" },
+                },
+                required: ["foundedYear"],
+              },
+            },
+            required: ["details"],
+          },
+        },
+        required: ["company"],
+      },
+      onSuccess: () => {},
+      onError: (error) => {
+        capturedError = error
+      },
+    })
+
+    // Missing deeply nested required field
+    await tool.execute!({ company: { details: {} } }, {
+      toolCallId: "test-call-id",
+      messages: [],
+      abortSignal: undefined as any,
+    })
+
+    expect(capturedError).toBeDefined()
+    // Error path should indicate the nested location
+    expect(capturedError).toContain("foundedYear")
+  })
+
  test("toModelOutput returns text value", () => {
    const tool = SessionPrompt.createStructuredOutputTool({
      schema: { type: "object" },
      onSuccess: () => {},
+      onError: () => {},
    })

    expect(tool.toModelOutput).toBeDefined()
@@ -253,4 +465,238 @@ describe("structured-output.createStructuredOutputTool", () => {
    expect(modelOutput.type).toBe("text")
    expect(modelOutput.value).toBe("Test output")
  })
+
+  // Tests for retry behavior simulation
+  describe("retry behavior", () => {
+    test("multiple validation failures trigger multiple onError calls", async () => {
+      let errorCount = 0
+      const errors: string[] = []
+
+      const tool = SessionPrompt.createStructuredOutputTool({
+        schema: {
+          type: "object",
+          properties: {
+            name: { type: "string" },
+            age: { type: "number" },
+          },
+          required: ["name", "age"],
+        },
+        onSuccess: () => {},
+        onError: (error) => {
+          errorCount++
+          errors.push(error)
+        },
+      })
+
+      // First attempt - missing both required fields
+      await tool.execute!({}, {
+        toolCallId: "call-1",
+        messages: [],
+        abortSignal: undefined as any,
+      })
+      expect(errorCount).toBe(1)
+
+      // Second attempt - still missing age
+      await tool.execute!({ name: "Test" }, {
+        toolCallId: "call-2",
+        messages: [],
+        abortSignal: undefined as any,
+      })
+      expect(errorCount).toBe(2)
+
+      // Third attempt - wrong type for age
+      await tool.execute!({ name: "Test", age: "not a number" }, {
+        toolCallId: "call-3",
+        messages: [],
+        abortSignal: undefined as any,
+      })
+      expect(errorCount).toBe(3)
+
+      // Verify each error is descriptive
+      expect(errors.length).toBe(3)
+      errors.forEach(error => {
+        expect(error.length).toBeGreaterThan(0)
+      })
+    })
+
+    test("success after failures calls onSuccess (not onError)", async () => {
+      let successCalled = false
+      let errorCount = 0
+      let capturedOutput: unknown
+
+      const tool = SessionPrompt.createStructuredOutputTool({
+        schema: {
+          type: "object",
+          properties: {
+            value: { type: "number" },
+          },
+          required: ["value"],
+        },
+        onSuccess: (output) => {
+          successCalled = true
+          capturedOutput = output
+        },
+        onError: () => {
+          errorCount++
+        },
+      })
+
+      // First attempt - wrong type
+      const result1 = await tool.execute!({ value: "wrong" }, {
+        toolCallId: "call-1",
+        messages: [],
+        abortSignal: undefined as any,
+      })
+      expect(errorCount).toBe(1)
+      expect(successCalled).toBe(false)
+      expect(result1.output).toContain("Validation failed")
+
+      // Second attempt - correct
+      const result2 = await tool.execute!({ value: 42 }, {
+        toolCallId: "call-2",
+        messages: [],
+        abortSignal: undefined as any,
+      })
+      expect(errorCount).toBe(1) // Should not increment
+      expect(successCalled).toBe(true)
+      expect(capturedOutput).toEqual({ value: 42 })
+      expect(result2.output).toBe("Structured output captured successfully.")
+    })
+
+    test("error messages guide model to fix issues", async () => {
+      const tool = SessionPrompt.createStructuredOutputTool({
+        schema: {
+          type: "object",
+          properties: {
+            count: { type: "integer" },
+            items: {
+              type: "array",
+              items: { type: "string" },
+            },
+          },
+          required: ["count", "items"],
+        },
+        onSuccess: () => {},
+        onError: () => {},
+      })
+
+      // Invalid input
+      const result = await tool.execute!({ count: 3.5, items: [1, 2, 3] }, {
+        toolCallId: "call-1",
+        messages: [],
+        abortSignal: undefined as any,
+      })
+
+      // Error message should tell model to fix and retry
+      expect(result.output).toContain("Validation failed")
+      expect(result.output).toContain("call StructuredOutput again")
+    })
+
+    test("simulates retry state tracking (like prompt.ts does)", async () => {
+      // This test simulates how prompt.ts tracks retry state
+      let structuredOutput: unknown | undefined
+      let structuredOutputError: string | undefined
+      let structuredOutputRetries = 0
+      const maxRetries = 2
+
+      const tool = SessionPrompt.createStructuredOutputTool({
+        schema: {
+          type: "object",
+          properties: { answer: { type: "number" } },
+          required: ["answer"],
+        },
+        onSuccess: (output) => {
+          structuredOutput = output
+        },
+        onError: (error) => {
+          structuredOutputError = error
+          structuredOutputRetries++
+        },
+      })
+
+      // Simulate retry loop like in prompt.ts
+      const attempts: Array<{ input: unknown; shouldRetry: boolean }> = [
+        { input: { answer: "wrong" }, shouldRetry: true },  // Attempt 1: fails
+        { input: { answer: "still wrong" }, shouldRetry: true },  // Attempt 2: fails
+        { input: { answer: "nope" }, shouldRetry: false },  // Attempt 3: fails, max exceeded
+      ]
+
+      for (const { input, shouldRetry } of attempts) {
+        await tool.execute!(input, {
+          toolCallId: `call-${structuredOutputRetries + 1}`,
+          messages: [],
+          abortSignal: undefined as any,
+        })
+
+        // Check if we should continue (like prompt.ts loop logic)
+        if (structuredOutput !== undefined) {
+          break // Success - exit loop
+        }
+
+        if (structuredOutputError) {
+          if (structuredOutputRetries <= maxRetries) {
+            expect(shouldRetry).toBe(true)
+            structuredOutputError = undefined // Reset for next attempt
+          } else {
+            expect(shouldRetry).toBe(false)
+            // Max retries exceeded - would set StructuredOutputError in prompt.ts
+            break
+          }
+        }
+      }
+
+      // Verify final state after max retries exceeded
+      expect(structuredOutputRetries).toBe(3)
+      expect(structuredOutput).toBeUndefined()
+    })
+
+    test("simulates successful retry after initial failures", async () => {
+      let structuredOutput: unknown | undefined
+      let structuredOutputError: string | undefined
+      let structuredOutputRetries = 0
+      const maxRetries = 2
+
+      const tool = SessionPrompt.createStructuredOutputTool({
+        schema: {
+          type: "object",
+          properties: { value: { type: "number" } },
+          required: ["value"],
+        },
+        onSuccess: (output) => {
+          structuredOutput = output
+        },
+        onError: (error) => {
+          structuredOutputError = error
+          structuredOutputRetries++
+        },
+      })
+
+      // Simulate: fail twice, then succeed on third attempt
+      const attempts = [
+        { value: "wrong" },      // Fails
+        { value: "also wrong" }, // Fails
+        { value: 42 },           // Succeeds
+      ]
+
+      for (const input of attempts) {
+        await tool.execute!(input, {
+          toolCallId: `call-${structuredOutputRetries + 1}`,
+          messages: [],
+          abortSignal: undefined as any,
+        })
+
+        if (structuredOutput !== undefined) {
+          break // Success
+        }
+
+        if (structuredOutputError && structuredOutputRetries <= maxRetries) {
+          structuredOutputError = undefined
+        }
+      }
+
+      // Should have succeeded on retry 2 (within maxRetries)
+      expect(structuredOutput).toEqual({ value: 42 })
+      expect(structuredOutputRetries).toBe(2) // Two failures before success
+    })
+  })
 })