Compare commits

...

2 Commits

Author SHA1 Message Date
Kit Langton
7aeb0972aa test(e2e): share dock child session mock 2026-03-16 12:39:32 -04:00
Kit Langton
eb5c67de58 fix(e2e): replace LLM-seeded question tests with route mocking
The question dock e2e tests used seedSessionQuestion which sends a
prompt to a real LLM and waits for it to call the question tool. This
is inherently flaky due to LLM latency and non-determinism.

Add withMockQuestion (mirroring the existing withMockPermission pattern)
that intercepts GET /question and POST /question/*/reply at the
Playwright route level, making the tests fully deterministic.
2026-03-16 12:16:16 -04:00

View File

@@ -5,7 +5,7 @@ import {
type ComposerProbeState,
type ComposerWindow,
} from "../../src/testing/session-composer"
import { cleanupSession, clearSessionDockSeed, seedSessionQuestion } from "../actions"
import { cleanupSession } from "../actions"
import {
permissionDockSelector,
promptSelector,
@@ -13,9 +13,11 @@ import {
sessionComposerDockSelector,
sessionTodoToggleButtonSelector,
} from "../selectors"
import { createSdk } from "../utils"
type Sdk = Parameters<typeof clearSessionDockSeed>[0]
type Sdk = ReturnType<typeof createSdk>
type PermissionRule = { permission: string; pattern: string; action: "allow" | "deny" | "ask" }
type Child = { id: string }
async function withDockSession<T>(
sdk: Sdk,
@@ -36,14 +38,6 @@ async function withDockSession<T>(
test.setTimeout(120_000)
async function withDockSeed<T>(sdk: Sdk, sessionID: string, fn: () => Promise<T>) {
try {
return await fn()
} finally {
await clearSessionDockSeed(sdk, sessionID).catch(() => undefined)
}
}
async function clearPermissionDock(page: any, label: RegExp) {
const dock = page.locator(permissionDockSelector)
await expect(dock).toBeVisible()
@@ -79,6 +73,91 @@ async function expectPermissionOpen(page: any) {
await expect(page.locator(promptSelector)).toBeVisible()
}
async function withMockSession<T>(page: any, child: Child | undefined, fn: () => Promise<T>) {
if (!child) return await fn()
const list = async (route: any) => {
const res = await route.fetch()
const json = await res.json()
const list = Array.isArray(json) ? json : Array.isArray(json?.data) ? json.data : undefined
if (Array.isArray(list) && !list.some((item) => item?.id === child.id)) list.push(child)
await route.fulfill({
status: res.status(),
headers: res.headers(),
contentType: "application/json",
body: JSON.stringify(json),
})
}
await page.route("**/session?*", list)
try {
return await fn()
} finally {
await page.unroute("**/session?*", list)
}
}
async function withMockQuestion<T>(
page: any,
request: {
id: string
sessionID: string
questions: Array<{
header: string
question: string
options: Array<{ label: string; description: string }>
multiple?: boolean
custom?: boolean
}>
},
child: Child | undefined,
fn: (state: { resolved: () => Promise<void> }) => Promise<T>,
) {
let pending = [
{
id: request.id,
sessionID: request.sessionID,
questions: request.questions,
},
]
const list = async (route: any) => {
await route.fulfill({
status: 200,
contentType: "application/json",
body: JSON.stringify(pending),
})
}
const reply = async (route: any) => {
const url = new URL(route.request().url())
const id = url.pathname.split("/").at(-2)
pending = pending.filter((item) => item.id !== id)
await route.fulfill({
status: 200,
contentType: "application/json",
body: JSON.stringify(true),
})
}
await page.route("**/question", list)
await page.route("**/question/*/reply", reply)
const state = {
async resolved() {
await expect.poll(() => pending.length, { timeout: 10_000 }).toBe(0)
},
}
try {
return await withMockSession(page, child, () => fn(state))
} finally {
await page.unroute("**/question", list)
await page.unroute("**/question/*/reply", reply)
}
}
async function todoDock(page: any, sessionID: string) {
await page.addInitScript(() => {
const win = window as ComposerWindow
@@ -183,7 +262,7 @@ async function withMockPermission<T>(
metadata?: Record<string, unknown>
always?: string[]
},
opts: { child?: any } | undefined,
child: Child | undefined,
fn: (state: { resolved: () => Promise<void> }) => Promise<T>,
) {
let pending = [
@@ -216,23 +295,6 @@ async function withMockPermission<T>(
await page.route("**/permission", list)
await page.route("**/session/*/permissions/*", reply)
const sessionList = opts?.child
? async (route: any) => {
const res = await route.fetch()
const json = await res.json()
const list = Array.isArray(json) ? json : Array.isArray(json?.data) ? json.data : undefined
if (Array.isArray(list) && !list.some((item) => item?.id === opts.child?.id)) list.push(opts.child)
await route.fulfill({
status: res.status(),
headers: res.headers(),
contentType: "application/json",
body: JSON.stringify(json),
})
}
: undefined
if (sessionList) await page.route("**/session?*", sessionList)
const state = {
async resolved() {
await expect.poll(() => pending.length, { timeout: 10_000 }).toBe(0)
@@ -240,11 +302,10 @@ async function withMockPermission<T>(
}
try {
return await fn(state)
return await withMockSession(page, child, () => fn(state))
} finally {
await page.unroute("**/permission", list)
await page.unroute("**/session/*/permissions/*", reply)
if (sessionList) await page.unroute("**/session?*", sessionList)
}
}
@@ -275,10 +336,12 @@ test("auto-accept toggle works before first submit", async ({ page, gotoSession
test("blocked question flow unblocks after submit", async ({ page, sdk, gotoSession }) => {
await withDockSession(sdk, "e2e composer dock question", async (session) => {
await withDockSeed(sdk, session.id, async () => {
await gotoSession(session.id)
await gotoSession(session.id)
await seedSessionQuestion(sdk, {
await withMockQuestion(
page,
{
id: "que_e2e_question",
sessionID: session.id,
questions: [
{
@@ -290,16 +353,20 @@ test("blocked question flow unblocks after submit", async ({ page, sdk, gotoSess
],
},
],
})
},
undefined,
async (state) => {
await page.goto(page.url())
await expectQuestionBlocked(page)
const dock = page.locator(questionDockSelector)
await expectQuestionBlocked(page)
await dock.locator('[data-slot="question-option"]').first().click()
await dock.getByRole("button", { name: /submit/i }).click()
await expectQuestionOpen(page)
})
const dock = page.locator(questionDockSelector)
await dock.locator('[data-slot="question-option"]').first().click()
await dock.getByRole("button", { name: /submit/i }).click()
await state.resolved()
await page.goto(page.url())
await expectQuestionOpen(page)
},
)
})
})
@@ -400,8 +467,10 @@ test("child session question request blocks parent dock and unblocks after submi
if (!child?.id) throw new Error("Child session create did not return an id")
try {
await withDockSeed(sdk, child.id, async () => {
await seedSessionQuestion(sdk, {
await withMockQuestion(
page,
{
id: "que_e2e_child_question",
sessionID: child.id,
questions: [
{
@@ -413,16 +482,20 @@ test("child session question request blocks parent dock and unblocks after submi
],
},
],
})
},
child,
async (state) => {
await page.goto(page.url())
await expectQuestionBlocked(page)
const dock = page.locator(questionDockSelector)
await expectQuestionBlocked(page)
await dock.locator('[data-slot="question-option"]').first().click()
await dock.getByRole("button", { name: /submit/i }).click()
await expectQuestionOpen(page)
})
const dock = page.locator(questionDockSelector)
await dock.locator('[data-slot="question-option"]').first().click()
await dock.getByRole("button", { name: /submit/i }).click()
await state.resolved()
await page.goto(page.url())
await expectQuestionOpen(page)
},
)
} finally {
await cleanupSession({ sdk, sessionID: child.id })
}
@@ -456,7 +529,7 @@ test("child session permission request blocks parent dock and supports allow onc
patterns: ["/tmp/opencode-e2e-perm-child"],
metadata: { description: "Need child permission" },
},
{ child },
child,
async (state) => {
await page.goto(page.url())
await expectPermissionBlocked(page)
@@ -506,10 +579,12 @@ test("todo dock transitions and collapse behavior", async ({ page, sdk, gotoSess
test("keyboard focus stays off prompt while blocked", async ({ page, sdk, gotoSession }) => {
await withDockSession(sdk, "e2e composer dock keyboard", async (session) => {
await withDockSeed(sdk, session.id, async () => {
await gotoSession(session.id)
await gotoSession(session.id)
await seedSessionQuestion(sdk, {
await withMockQuestion(
page,
{
id: "que_e2e_keyboard",
sessionID: session.id,
questions: [
{
@@ -518,13 +593,16 @@ test("keyboard focus stays off prompt while blocked", async ({ page, sdk, gotoSe
options: [{ label: "Continue", description: "Continue now" }],
},
],
})
},
undefined,
async () => {
await page.goto(page.url())
await expectQuestionBlocked(page)
await expectQuestionBlocked(page)
await page.locator("main").click({ position: { x: 5, y: 5 } })
await page.keyboard.type("abc")
await expect(page.locator(promptSelector)).toHaveCount(0)
})
await page.locator("main").click({ position: { x: 5, y: 5 } })
await page.keyboard.type("abc")
await expect(page.locator(promptSelector)).toHaveCount(0)
},
)
})
})