diff --git a/apps/extension/src/tools/__tests__/dispatcher.test.ts b/apps/extension/src/tools/__tests__/dispatcher.test.ts index 5407a2cc..1c43515c 100644 --- a/apps/extension/src/tools/__tests__/dispatcher.test.ts +++ b/apps/extension/src/tools/__tests__/dispatcher.test.ts @@ -603,6 +603,7 @@ describe("ToolDispatcher", () => { "focus", "blur", "scroll_to", + "wheel", ] as const)("routes %s with hover cleanup and cooperative cancellation", async (action) => { const tab = { id: 7, windowId: 4242, active: true }; vi.stubGlobal("chrome", { @@ -630,6 +631,9 @@ describe("ToolDispatcher", () => { return new Promise((resolve) => { resolveNode = resolve; }); + if (method === "DOM.getContentQuads") return { quads: [[0, 0, 100, 0, 100, 100, 0, 100]] }; + if (method === "Page.getLayoutMetrics") + return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 } }; if (method === "Runtime.callFunctionOn") return { result: { value: true } }; return {}; }), @@ -642,7 +646,13 @@ describe("ToolDispatcher", () => { hover.rememberHover("aa11", { tab_id: 7, x: 10, y: 20 }); await hover.setHoverBypass("aa11", 7, true); dispatcher.start(); - deliver(makeRequest(`tool.${action}`, { session_id: "aa11", ref: "e1" })); + deliver( + makeRequest(`tool.${action}`, { + session_id: "aa11", + ref: "e1", + ...(action === "wheel" ? { delta_y: 120 } : {}), + }), + ); await vi.waitFor(() => expect(resolveNode).toBeDefined()); expect(onBrowserControlResumed).toHaveBeenCalledWith("aa11"); deliver({ id: "cancel-focus", method: "cancel", params: { rpc_id: "r-1" } }); @@ -655,7 +665,7 @@ describe("ToolDispatcher", () => { }); expect(cdp.send).not.toHaveBeenCalledWith(7, "DOM.focus", expect.anything()); expect(cdp.send).not.toHaveBeenCalledWith(7, "Runtime.callFunctionOn", expect.anything()); - if (action === "scroll_to") { + if (action === "scroll_to" || action === "wheel") { expect(cdp.send).toHaveBeenCalledWith(7, "Runtime.releaseObjectGroup", { objectGroup: expect.any(String), }); @@ -668,6 +678,18 @@ describe("ToolDispatcher", () => { 7, expect.objectContaining({ enabled: false }), ); + if (action === "wheel") { + expect(cdp.send).toHaveBeenCalledWith(7, "Input.dispatchMouseEvent", { + type: "mouseMoved", + x: 10, + y: 20, + }); + expect(cdp.send).not.toHaveBeenCalledWith( + 7, + "Input.dispatchMouseEvent", + expect.objectContaining({ type: "mouseWheel" }), + ); + } expect(dispatcher.inflightAbortControllers.size).toBe(0); dispatcher.stop(); }); diff --git a/apps/extension/src/tools/__tests__/wheel.browser.test.ts b/apps/extension/src/tools/__tests__/wheel.browser.test.ts new file mode 100644 index 00000000..7b8577c1 --- /dev/null +++ b/apps/extension/src/tools/__tests__/wheel.browser.test.ts @@ -0,0 +1,320 @@ +// @vitest-environment node +// Opt in with BSK_WHEEL_CHROME=/path/to/chrome. Uses an isolated browser/profile. +import { describe, expect, it } from "vitest"; +import type { CdpFrame, CdpFrameGraph, CdpTarget } from "@/browser-driver/frame-graph"; +import { SessionManager } from "@/session-manager/manager"; +import type { WheelParams } from "@/transport/types"; +import type { CdpRunner } from "../shared"; +import { handleWheel } from "../wheel"; + +type Send = >( + method: string, + params?: object, + sessionId?: string, +) => Promise; +type Tree = { frame: { id: string; parentId?: string; name?: string }; childFrames?: Tree[] }; + +async function withBrowser( + run: (h: Awaited>) => Promise, + zoom = 1, +) { + const { withChrome } = await import( + new URL( + "../../../../../evals/browser/cases/regression/snapshot-coordinates/chrome.mjs", + import.meta.url, + ).href + ); + await withChrome( + { executable: process.env.BSK_WHEEL_CHROME, deviceScale: 1, zoom }, + async (send: Send) => run(await harness(send)), + ); +} + +async function harness(send: Send) { + const { targetId } = await send<{ targetId: string }>("Target.createTarget", { + url: "about:blank", + }); + const { sessionId: rootSession } = await send<{ sessionId: string }>("Target.attachToTarget", { + targetId, + flatten: true, + }); + await send("Page.bringToFront", {}, rootSession); + const manager = new SessionManager({ + agentWindow: { + create: async () => 100, + remove: async () => {}, + ensureActiveTab: async () => 4, + }, + }); + const ctx = await manager.start("aa11"); + const calls: { method: string; sessionId?: string }[] = []; + const after = { command: (_method: string) => {} }; + const sessionFor = (target: CdpTarget) => target.sessionId ?? rootSession; + const forward = async (target: CdpTarget, method: string, params?: object): Promise => { + calls.push({ method, sessionId: target.sessionId }); + const result = await send(method, params, sessionFor(target)); + after.command(method); + return result; + }; + const cdp: CdpRunner = { + send: (_tab, method, params) => forward({ tabId: 4 }, method, params), + sendToTarget: forward, + }; + const evaluate = async (expression: string, frame?: CdpFrame): Promise => { + const session = frame ? sessionFor(frame.target) : rootSession; + const world = frame + ? await send<{ executionContextId: number }>( + "Page.createIsolatedWorld", + { + frameId: frame.frameId, + worldName: "wheel-oracle", + }, + session, + ) + : undefined; + const reply = await send<{ result: { value: T }; exceptionDetails?: unknown }>( + "Runtime.evaluate", + { + expression, + contextId: world?.executionContextId, + returnByValue: true, + awaitPromise: true, + }, + session, + ); + expect(reply.exceptionDetails).toBeUndefined(); + return reply.result.value; + }; + const rememberRef = (frame: CdpFrame, backendNodeId: number) => + ctx.refStore.set("e1", backendNodeId, { + tabId: 4, + frameId: frame.frameId, + cdpSessionId: frame.target.sessionId, + }); + const ref = async (frame: CdpFrame, selector = "#probe") => { + const { executionContextId } = await send<{ executionContextId: number }>( + "Page.createIsolatedWorld", + { + frameId: frame.frameId, + worldName: "wheel-oracle", + }, + sessionFor(frame.target), + ); + const { result } = await send<{ result: { objectId: string } }>( + "Runtime.evaluate", + { + expression: `document.querySelector(${JSON.stringify(selector)})`, + contextId: executionContextId, + }, + sessionFor(frame.target), + ); + const { node } = await send<{ node: { backendNodeId: number } }>( + "DOM.describeNode", + { objectId: result.objectId }, + sessionFor(frame.target), + ); + await send("Runtime.releaseObject", { objectId: result.objectId }, sessionFor(frame.target)); + rememberRef(frame, node.backendNodeId); + }; + const wheel = (params: Partial, signal?: AbortSignal) => + handleWheel( + manager, + { session_id: "aa11", ...params }, + { + cdp, + signal, + tabsApi: { + get: async (id) => ({ id, windowId: 100, active: true }) as chrome.tabs.Tab, + query: async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab], + }, + }, + ); + const loadFrames = async () => { + const { targetInfos } = await send<{ targetInfos: { targetId: string; type: string }[] }>( + "Target.getTargets", + ); + const sessions = [rootSession]; + for (const target of targetInfos.filter((t) => t.type === "iframe")) { + const { sessionId } = await send<{ sessionId: string }>("Target.attachToTarget", { + targetId: target.targetId, + flatten: true, + }); + sessions.push(sessionId); + } + const frames: (CdpFrame & { name: string })[] = []; + for (const session of sessions) { + const { frameTree } = await send<{ frameTree: Tree }>("Page.getFrameTree", {}, session); + const target = { tabId: 4, ...(session !== rootSession ? { sessionId: session } : {}) }; + const visit = (tree: Tree, parentFrameId = tree.frame.parentId) => { + frames.push({ frameId: tree.frame.id, parentFrameId, target, name: tree.frame.name ?? "" }); + for (const child of tree.childFrames ?? []) visit(child, tree.frame.id); + }; + visit(frameTree); + } + for (const frame of frames) { + if (!frame.parentFrameId) continue; + const parent = frames.find((f) => f.frameId === frame.parentFrameId)!; + const owner = await send<{ backendNodeId: number }>( + "DOM.getFrameOwner", + { frameId: frame.frameId }, + sessionFor(parent.target), + ); + frame.ownerBackendNodeId = owner.backendNodeId; + } + const graph: CdpFrameGraph = { rootFrameId: frames[0].frameId, frames }; + cdp.getFrameGraph = async () => graph; + return frames; + }; + return { send, rootSession, evaluate, ref, rememberRef, wheel, loadFrames, calls, after }; +} + +const LISTEN = `window.wheels = []; window.blockWheel = false; + document.addEventListener('wheel', event => { + window.wheels.push({ trusted:event.isTrusted, dx:event.deltaX, dy:event.deltaY, + mode:event.deltaMode, ctrl:event.ctrlKey, shift:event.shiftKey, + ids:event.composedPath().map(node=>node.id).filter(Boolean) }); + if (window.blockWheel) event.preventDefault(); + }, { capture:true, passive:false });`; + +describe.skipIf(!process.env.BSK_WHEEL_CHROME)("real browser wheel", () => { + it("sends trusted viewport input in both directions and horizontally", async () => { + await withBrowser(async (h) => { + await h.evaluate( + `document.body.style.cssText='margin:0;width:4000px;height:6000px'; ${LISTEN}`, + ); + expect(await h.wheel({ delta_y: 300 })).toMatchObject({ delta_x: 0, delta_y: 300 }); + await expect.poll(() => h.evaluate("scrollY")).toBe(300); + expect(await h.wheel({ delta_y: -100 })).not.toHaveProperty("code"); + await expect.poll(() => h.evaluate("scrollY")).toBe(200); + expect(await h.wheel({ delta_x: 200 })).not.toHaveProperty("code"); + await expect.poll(() => h.evaluate("scrollX")).toBe(200); + expect( + await h.evaluate("window.wheels.map(({trusted,dx,dy,mode})=>({trusted,dx,dy,mode}))"), + ).toEqual([ + { trusted: true, dx: 0, dy: 300, mode: 0 }, + { trusted: true, dx: 0, dy: -100, mode: 0 }, + { trusted: true, dx: 200, dy: 0, mode: 0 }, + ]); + }); + }); + + it.each([ + 1, 1.25, + ])("targets a nested scroller and respects preventDefault at zoom %s", async (zoom) => { + await withBrowser(async (h) => { + await h.evaluate( + `document.body.innerHTML = '
'; ${LISTEN}`, + ); + expect(await h.wheel({ selector: "#panel", delta_y: 120 })).not.toHaveProperty("code"); + await expect + .poll(() => h.evaluate("document.querySelector('#panel').scrollTop")) + .toBeCloseTo(120 / zoom, 0); + expect(await h.evaluate("scrollY")).toBe(0); + expect(await h.evaluate("window.wheels[0].ids")).toContain("panel"); + const before = await h.evaluate("document.querySelector('#panel').scrollTop"); + await h.evaluate("window.blockWheel=true; window.wheels=[]"); + expect( + await h.wheel({ + selector: "#panel", + delta_x: 25, + delta_y: -50, + modifiers: ["ctrl", "shift"], + }), + ).not.toHaveProperty("code"); + await expect.poll(() => h.evaluate("window.wheels.length")).toBe(1); + const event = await h.evaluate<{ dx: number; dy: number }>("window.wheels[0]"); + expect(event).toMatchObject({ trusted: true, ctrl: true, shift: true }); + // Native Chromium adjusts wheel deltas by page zoom before DOM delivery. + expect(event.dx).toBeCloseTo(25 / zoom, 3); + expect(event.dy).toBeCloseTo(-50 / zoom, 3); + await h.evaluate( + "new Promise(resolve=>requestAnimationFrame(()=>requestAnimationFrame(resolve)))", + ); + expect(await h.evaluate("document.querySelector('#panel').scrollTop")).toBe(before); + }, zoom); + }); + + it("hits the visible portion of a target clipped by a small ancestor", async () => { + await withBrowser(async (h) => { + await h.evaluate( + `document.body.innerHTML='
'; ${LISTEN}`, + ); + expect(await h.wheel({ selector: "#probe", delta_y: 100 })).not.toHaveProperty("code"); + await expect.poll(() => h.evaluate("window.wheels.length")).toBe(1); + expect(await h.evaluate("window.wheels[0].ids")).toContain("probe"); + await expect + .poll(() => h.evaluate("document.querySelector('#clip').scrollTop")) + .toBeGreaterThan(0); + }); + }); + + it.each([ + { fixture: "snapshot-coordinates", zoom: 1 }, + { fixture: "snapshot-coordinates", zoom: 1.25 }, + { fixture: "oopif-scrollbars", zoom: 0.8 }, + ])("routes wheel to nested frames: $fixture at zoom $zoom", async ({ fixture, zoom }) => { + const { createEvalServer } = await import( + new URL("../../../../../evals/browser/lib/server.mjs", import.meta.url).href + ); + const server = createEvalServer(); + const { baseUrl } = await server.start(); + try { + await withBrowser(async (h) => { + await h.send("Page.navigate", { url: `${baseUrl}/${fixture}?run=wheel` }, h.rootSession); + await expect + .poll( + () => + server + .snapshot("wheel") + .events.some( + (e: { type: string; data: { root?: boolean } }) => + e.type === "geometry.ready" && e.data.root, + ), + { timeout: 10_000 }, + ) + .toBe(true); + const frames = await h.loadFrames(); + expect(frames.length).toBe(fixture === "snapshot-coordinates" ? 5 : 3); + expect(frames.some((frame) => frame.target.sessionId)).toBe(true); + for (const frame of frames) await h.evaluate(`${LISTEN} window.blockWheel=true`, frame); + for (const frame of frames) { + for (const other of frames) await h.evaluate("window.wheels=[]", other); + await h.ref(frame); + expect(await h.wheel({ ref: "e1", delta_y: 90 })).not.toHaveProperty("code"); + await expect.poll(() => h.evaluate("window.wheels.length", frame)).toBe(1); + const event = await h.evaluate<{ dy: number }>("window.wheels[0]", frame); + expect(event).toMatchObject({ + trusted: true, + dx: 0, + ids: expect.arrayContaining(["probe"]), + }); + expect(event.dy).toBeCloseTo(90 / zoom, 3); + for (const other of frames.filter((other) => other !== frame)) + expect(await h.evaluate("window.wheels.length", other)).toBe(0); + } + }, zoom); + } finally { + await server.stop(); + } + }, 30_000); + + it("cancels during selector lookup before scrolling or input", async () => { + await withBrowser(async (h) => { + await h.evaluate( + `document.body.innerHTML='
'; ${LISTEN}`, + ); + const abort = new AbortController(); + h.after.command = (method) => { + if (method === "DOM.getDocument") abort.abort(); + }; + expect(await h.wheel({ selector: "#probe", delta_y: 100 }, abort.signal)).toMatchObject({ + code: "cancelled", + }); + expect(h.calls.map((c) => c.method)).toEqual(["DOM.getDocument"]); + expect(await h.evaluate("({y:scrollY,count:window.wheels.length})")).toEqual({ + y: 0, + count: 0, + }); + }); + }); +}); diff --git a/apps/extension/src/tools/__tests__/wheel.test.ts b/apps/extension/src/tools/__tests__/wheel.test.ts new file mode 100644 index 00000000..f2ca6dfc --- /dev/null +++ b/apps/extension/src/tools/__tests__/wheel.test.ts @@ -0,0 +1,422 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import type { CdpFrameGraph, CdpTarget } from "@/browser-driver/frame-graph"; +import { SessionManager } from "@/session-manager/manager"; +import type { JavaScriptDialogInfo, WheelParams } from "@/transport/types"; +import type { ViewportRect } from "../geometry"; +import type { CdpRunner } from "../shared"; +import { handleWheel } from "../wheel"; + +interface Call { + target: CdpTarget; + method: string; + params?: Record; + aborted: boolean; +} + +async function fixture(frame: "top" | "same-target" | "oopif" = "top") { + const manager = new SessionManager({ + agentWindow: { + create: async () => 100, + remove: async () => {}, + ensureActiveTab: async () => 4, + }, + }); + const ctx = await manager.start("aa11"); + ctx.refStore.set("e3", 1234, { + tabId: 4, + ...(frame !== "top" ? { frameId: "child" } : {}), + ...(frame === "oopif" ? { cdpSessionId: "child-session" } : {}), + }); + const abort = new AbortController(); + const graph: CdpFrameGraph = { + rootFrameId: "main", + frames: [ + { frameId: "main", target: { tabId: 4 } }, + { + frameId: "child", + parentFrameId: "main", + ownerBackendNodeId: 99, + target: { tabId: 4, ...(frame === "oopif" ? { sessionId: "child-session" } : {}) }, + }, + ], + }; + const rectangles = new Map([ + [1234, { x: 10, y: 20, width: 100, height: 40 }], + [99, { x: 204, y: 306, width: 400, height: 200 }], + ]); + const calls: Call[] = []; + const hooks = { + after: (_call: Call) => {}, + reply: (_call: Call): object | undefined => undefined, + }; + const send = async (target: CdpTarget, method: string, params?: object): Promise => { + const call = { + target, + method, + params: params as Call["params"], + aborted: abort.signal.aborted, + }; + calls.push(call); + const override = hooks.reply(call); + hooks.after(call); + if (override) return override as T; + const args = call.params ?? {}; + const replies: Record = { + "DOM.getDocument": { root: { nodeId: 1 } }, + "DOM.querySelector": { nodeId: 2 }, + "DOM.describeNode": { node: { backendNodeId: 1234 } }, + "DOM.scrollIntoViewIfNeeded": {}, + "Input.dispatchMouseEvent": {}, + "DOM.getContentQuads": { + quads: [ + frame === "same-target" + ? [224, 346, 424, 346, 424, 426, 224, 426] + : [10, 20, 110, 20, 110, 60, 10, 60], + ], + }, + "DOM.resolveNode": { object: { objectId: String(args.backendNodeId) } }, + "Runtime.callFunctionOn": { + result: { + value: String(args.functionDeclaration).includes("IntersectionObserver") + ? rectangles.get(Number(args.objectId)) + : { width: 200, height: 100 }, + }, + }, + "Runtime.releaseObject": {}, + "Runtime.releaseObjectGroup": {}, + "Runtime.evaluate": { result: { value: { width: 200, height: 100 } } }, + "Page.getLayoutMetrics": { + cssLayoutViewport: { + clientWidth: target.sessionId ? 185 : 1280, + clientHeight: target.sessionId ? 89 : 720, + }, + }, + "DOM.getBoxModel": { model: { content: [204, 306, 604, 306, 604, 506, 204, 506] } }, + }; + if (!(method in replies)) throw new Error(`unexpected CDP call ${method}`); + return replies[method] as T; + }; + const cdp: CdpRunner = { + send: (tabId, method, params) => send({ tabId }, method, params), + sendToTarget: send, + getFrameGraph: vi.fn(async () => { + hooks.after({ target: { tabId: 4 }, method: "getFrameGraph", aborted: abort.signal.aborted }); + return graph; + }), + trackSessionTab: vi.fn(), + }; + const tabsApi = { + get: vi.fn(async (id: number) => ({ id, windowId: 100, active: true }) as chrome.tabs.Tab), + query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]), + }; + const bypassOverlay = vi.fn(async (_tab: number, _enabled: boolean) => {}); + const run = (params: Partial = {}) => + handleWheel( + manager, + { session_id: "aa11", delta_y: 120, ...params }, + { cdp, tabsApi, signal: abort.signal, bypassOverlay }, + ); + return { ctx, cdp, graph, rectangles, calls, hooks, abort, tabsApi, bypassOverlay, run }; +} + +afterEach(() => vi.restoreAllMocks()); + +describe("handleWheel", () => { + it("moves to the viewport centre and sends signed native deltas and modifiers", async () => { + const f = await fixture(); + expect( + await f.run({ delta_x: -20.5, delta_y: 600, modifiers: ["ctrl", "shift"] }), + ).toMatchObject({ + tab_id: 4, + x: 640, + y: 360, + delta_x: -20.5, + delta_y: 600, + }); + expect(f.calls.map(({ method, params }) => ({ method, params }))).toEqual([ + { method: "Page.getLayoutMetrics", params: {} }, + { + method: "Input.dispatchMouseEvent", + params: { type: "mouseMoved", x: 640, y: 360, modifiers: 10 }, + }, + { + method: "Input.dispatchMouseEvent", + params: { type: "mouseWheel", x: 640, y: 360, deltaX: -20.5, deltaY: 600, modifiers: 10 }, + }, + ]); + expect(f.bypassOverlay.mock.calls).toEqual([ + [4, true], + [4, false], + ]); + }); + + it("supports pure horizontal input", async () => { + const f = await fixture(); + expect(await f.run({ delta_x: -120, delta_y: undefined })).toMatchObject({ + delta_x: -120, + delta_y: 0, + }); + }); + + it.each([ + { ref: "@e3" }, + { selector: "#panel" }, + ])("resolves %j and cleans up measurement objects", async (target) => { + const f = await fixture(); + expect(await f.run(target)).toMatchObject({ + x: 60, + y: 40, + delta_y: 120, + ...(target.ref ? { used_ref: "e3" } : { used_selector: "#panel" }), + }); + const group = f.calls.find((c) => c.method === "DOM.resolveNode")!.params!.objectGroup; + expect(f.calls.at(-1)).toMatchObject({ + method: "Runtime.releaseObjectGroup", + params: { objectGroup: group }, + }); + }); + + it.each([ + "same-target", + "oopif", + ] as const)("projects %s refs and routes input to the root target", async (frame) => { + const f = await fixture(frame); + expect(await f.run({ ref: "@e3" })).toMatchObject({ x: 324, y: 386 }); + expect( + f.calls + .filter((c) => c.method === "DOM.scrollIntoViewIfNeeded") + .map((c) => c.params!.backendNodeId), + ).toEqual([99, 1234]); + expect(f.cdp.getFrameGraph).toHaveBeenCalledTimes(1); + expect( + f.calls + .filter((c) => c.method === "Input.dispatchMouseEvent") + .every((c) => c.target.sessionId === undefined), + ).toBe(true); + if (frame === "oopif") { + expect(f.calls.find((c) => c.params?.backendNodeId === 1234)!.target.sessionId).toBe( + "child-session", + ); + expect( + f.calls + .filter((c) => c.method === "Runtime.releaseObjectGroup") + .map((c) => c.target.sessionId) + .sort(), + ).toEqual(["child-session", undefined].sort()); + } + }); + + it("chooses a point within an ordinary ancestor's clip", async () => { + const f = await fixture(); + f.rectangles.set(1234, { x: 10, y: 20, width: 20, height: 40 }); + expect(await f.run({ ref: "@e3" })).toMatchObject({ x: 20, y: 40 }); + }); + + it.each([ + 1234, 99, + ])("rejects a hidden or fully clipped node %i before sending input", async (node) => { + const f = await fixture("oopif"); + f.rectangles.set(node, null); + expect(await f.run({ ref: "@e3" })).toMatchObject({ + code: "permission_denied", + data: { reason: "element_not_visible" }, + }); + expect(f.calls.some((c) => c.method === "Input.dispatchMouseEvent")).toBe(false); + }); + + it.each([ + { delta_y: 0 }, + { delta_y: undefined }, + { delta_y: Number.NaN }, + { delta_x: Infinity }, + { ref: "" }, + { selector: " " }, + { ref: "@e3", selector: "#panel" }, + { timeout_ms: 0 }, + { timeout_ms: -1 }, + { timeout_ms: 1.5 }, + ])("rejects invalid params %j without browser calls", async (params) => { + const f = await fixture(); + expect(await f.run(params)).toMatchObject({ code: "invalid_params" }); + expect(f.calls).toEqual([]); + expect(f.tabsApi.query).not.toHaveBeenCalled(); + expect(f.bypassOverlay).not.toHaveBeenCalled(); + }); + + it.each([ + { ref: null }, + { selector: 42 }, + { delta_y: null }, + { delta_x: "10" }, + { timeout_ms: null }, + { modifiers: "ctrl" }, + { modifiers: ["invalid"] }, + ])("rejects malformed wire params %j", async (params) => { + const f = await fixture(); + expect(await f.run(params as unknown as Partial)).toMatchObject({ + code: "invalid_params", + }); + expect(f.calls).toEqual([]); + }); + + it.each(["stale", "other-tab"])("rejects %s refs", async (mode) => { + const f = await fixture(); + if (mode === "stale") f.ctx.refStore.clear(); + else f.ctx.refStore.set("e3", 1234, { tabId: 8 }); + expect(await f.run({ ref: "@e3" })).toMatchObject({ + code: "not_found", + data: { reason: "ref_not_found" }, + }); + expect(f.calls).toEqual([]); + }); + + it("reports missing selectors without sending input", async () => { + const f = await fixture(); + f.hooks.reply = (c) => (c.method === "DOM.querySelector" ? { nodeId: 0 } : undefined); + expect(await f.run({ selector: "#missing" })).toMatchObject({ + code: "not_found", + data: { reason: "selector_not_found" }, + }); + expect(f.calls.some((c) => c.method === "Input.dispatchMouseEvent")).toBe(false); + }); + + it("enforces the Agent Window boundary", async () => { + const f = await fixture(); + f.tabsApi.get.mockResolvedValue({ id: 4, windowId: 200, active: true } as chrome.tabs.Tab); + expect(await f.run({ tab_id: 4 })).toMatchObject({ code: "permission_denied" }); + expect(f.calls).toEqual([]); + }); + + it.each([ + "getFrameGraph", + "DOM.scrollIntoViewIfNeeded", + "DOM.getContentQuads", + "DOM.resolveNode", + "Runtime.callFunctionOn", + "Page.getLayoutMetrics", + "Runtime.evaluate", + "DOM.getBoxModel", + "Input.dispatchMouseEvent", + ])("stops at cancellation during %s, allowing only cleanup", async (method) => { + const f = await fixture("oopif"); + f.hooks.after = (c) => { + if (c.method === method) f.abort.abort(); + }; + expect(await f.run({ ref: "@e3" })).toMatchObject({ code: "cancelled" }); + expect(f.abort.signal.aborted).toBe(true); + expect(f.calls.filter((c) => c.aborted && c.method !== "Runtime.releaseObjectGroup")).toEqual( + [], + ); + expect(f.bypassOverlay.mock.calls.at(-1)?.[1] ?? false).toBe(false); + }); + + it.each([ + "DOM.getDocument", + "DOM.querySelector", + "DOM.describeNode", + ])("stops selector lookup after cancellation during %s", async (method) => { + const f = await fixture(); + f.hooks.after = (c) => { + if (c.method === method) f.abort.abort(); + }; + expect(await f.run({ selector: "#panel" })).toMatchObject({ code: "cancelled" }); + expect(f.calls.filter((c) => c.aborted)).toEqual([]); + }); + + it("does not start after early cancellation or cancellation during tab resolution", async () => { + for (const early of [true, false]) { + const f = await fixture(); + if (early) f.abort.abort(); + else + f.tabsApi.query.mockImplementation(async () => { + f.abort.abort(); + return [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]; + }); + expect(await f.run()).toMatchObject({ code: "cancelled" }); + expect(f.calls).toEqual([]); + } + }); + + it("does not run fallback scrolling after cancellation", async () => { + const f = await fixture(); + f.hooks.reply = (c) => { + if (c.method === "DOM.scrollIntoViewIfNeeded") { + f.abort.abort(); + throw new Error("scroll failed"); + } + return undefined; + }; + expect(await f.run({ ref: "@e3" })).toMatchObject({ code: "cancelled" }); + expect(f.calls.map((c) => c.method)).toEqual(["DOM.scrollIntoViewIfNeeded"]); + }); + + it.each([ + "DOM.scrollIntoViewIfNeeded", + "Page.getLayoutMetrics", + "Input.dispatchMouseEvent", + ])("stops before further input when the deadline expires during %s", async (method) => { + const f = await fixture("oopif"); + const now = vi.spyOn(Date, "now").mockReturnValue(100); + f.hooks.after = (c) => { + if (c.method === method) now.mockReturnValue(201); + }; + expect(await f.run({ ref: "@e3", timeout_ms: 100 })).toMatchObject({ code: "timeout" }); + expect(f.calls.some((c) => c.params?.type === "mouseWheel")).toBe(false); + expect(f.calls.filter((c) => c.method === method)).toHaveLength(1); + }); + + it.each([ + "cancel", + "timeout", + "failure", + ])("restores the overlay after %s while enabling bypass", async (mode) => { + const f = await fixture(); + const now = vi.spyOn(Date, "now").mockReturnValue(100); + f.bypassOverlay.mockImplementation(async (_tab, enabled) => { + if (!enabled) return; + if (mode === "cancel") f.abort.abort(); + if (mode === "timeout") now.mockReturnValue(201); + if (mode === "failure") throw new Error("overlay unavailable"); + }); + expect(await f.run({ timeout_ms: 100 })).toMatchObject({ + code: mode === "cancel" ? "cancelled" : mode === "timeout" ? "timeout" : "cdp_failed", + }); + expect(f.calls.some((c) => c.method === "Input.dispatchMouseEvent")).toBe(false); + expect(f.bypassOverlay.mock.calls).toEqual([ + [4, true], + [4, false], + ]); + }); + + it("preserves the browser error while cleaning up objects and overlay", async () => { + const f = await fixture(); + f.hooks.reply = (c) => { + if (c.params?.type === "mouseWheel") throw new Error("wheel failed"); + if (c.method === "Runtime.releaseObjectGroup") throw new Error("target closed"); + return undefined; + }; + f.bypassOverlay.mockImplementation(async (_tab, enabled) => { + if (!enabled) throw new Error("target closed"); + }); + expect(await f.run({ ref: "@e3" })).toMatchObject({ + code: "cdp_failed", + message: "wheel failed", + }); + expect(f.calls.at(-1)!.method).toBe("Runtime.releaseObjectGroup"); + expect(f.bypassOverlay.mock.calls).toEqual([ + [4, true], + [4, false], + ]); + }); + + it("attaches dialogs handled during input", async () => { + const f = await fixture(); + f.cdp.dialogCursor = () => 5; + const dialogs: JavaScriptDialogInfo[] = [ + { sequence: 6, tab_id: 4, type: "alert", message: "wheel", handled: "accepted" }, + ]; + f.cdp.dialogsSince = vi.fn(() => dialogs); + expect(await f.run()).toHaveProperty("dialogs", dialogs); + expect(f.cdp.dialogsSince).toHaveBeenCalledWith(4, 5); + }); +}); diff --git a/apps/extension/src/tools/dispatcher.ts b/apps/extension/src/tools/dispatcher.ts index 0415ca8e..f29ed5ed 100644 --- a/apps/extension/src/tools/dispatcher.ts +++ b/apps/extension/src/tools/dispatcher.ts @@ -34,6 +34,7 @@ import type { SnapshotParams, UploadParams, WaitForNavigationParams, + WheelParams, } from "@/transport/types"; import { isRequestFrame } from "@/transport/types"; import { handleConsole } from "./console"; @@ -98,6 +99,7 @@ import { } from "./tabs"; import { handleUpload } from "./upload"; import { handleWaitForNavigation } from "./waits"; +import { handleWheel } from "./wheel"; import { handleWindowResize, type WindowResizeParams } from "./window"; type DispatcherCdpRunner = CdpRunner & @@ -527,6 +529,20 @@ export class ToolDispatcher { ); return this.rememberHover((req.params as HoverParams).session_id, result); } + case "tool.wheel": + return this.withHoverReassert( + req.params as WheelParams, + () => + handleWheel( + this.sessions, + req.params as WheelParams, + this.cdp + ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal, bypassOverlay } + : undefined, + ), + { releaseAfter: true }, + signal, + ); case "tool.scroll_to": return this.withHoverReleaseForRequest( req.params as ScrollToParams, @@ -830,6 +846,7 @@ function sessionIdForBrowserControlMethod(req: RequestFrame): string | null { case "tool.reload": case "tool.click": case "tool.hover": + case "tool.wheel": case "tool.scroll_to": case "tool.focus": case "tool.blur": diff --git a/apps/extension/src/tools/scroll-visibility.ts b/apps/extension/src/tools/scroll-visibility.ts index a015f6d2..5511e177 100644 --- a/apps/extension/src/tools/scroll-visibility.ts +++ b/apps/extension/src/tools/scroll-visibility.ts @@ -107,7 +107,7 @@ async function projectRect( return projectRegionToViewport(region, projection); } -/** Only scroll-to needs renderer-clipped bounds; other interaction geometry is unchanged. */ +/** Renderer-clipped bounds for scroll-to and wheel; shared interaction geometry is unchanged. */ export async function scrollVisibleBounds( cdp: CdpRunner, tabId: number, diff --git a/apps/extension/src/tools/wheel.ts b/apps/extension/src/tools/wheel.ts new file mode 100644 index 00000000..512de715 --- /dev/null +++ b/apps/extension/src/tools/wheel.ts @@ -0,0 +1,246 @@ +import { ChromiumCdp } from "@/browser-driver/chromium-cdp"; +import { type CdpTarget, cdpTargetKey } from "@/browser-driver/frame-graph"; +import type { SessionContext, SessionManager } from "@/session-manager/manager"; +import type { RpcError, WheelParams, WheelResult } from "@/transport/types"; +import { attachDialogs, markDialogCursor } from "./dialogs"; +import { cdpError, rpcError } from "./errors"; +import { resolveNodeGeometry } from "./frame-geometry"; +import { clipPolygon, polygonArea, polygonCentroid, rectPolygon } from "./geometry"; +import { modifiersBitfield, resolveBackendNode } from "./interaction"; +import { scrollVisibleBounds } from "./scroll-visibility"; +import { + type CdpRunner, + type ChromeTabsApi, + chromeTabsApi, + enforceAgentWindow, + isRpcError, + lookupSession, + resolveTargetTab, + sendToCdpTarget, +} from "./shared"; + +export interface WheelDeps { + cdp: CdpRunner; + tabsApi: ChromeTabsApi; + signal?: AbortSignal; + /** Temporarily disable the Agent Window overlay's input blocker. */ + bypassOverlay?: (tabId: number, enabled: boolean) => Promise; +} + +interface WheelPoint { + x: number; + y: number; + usedRef?: string; + usedSelector?: string; +} + +let defaultDeps: { cdp: ChromiumCdp; tabsApi: ChromeTabsApi } | null = null; +function getDefaultDeps(): { cdp: ChromiumCdp; tabsApi: ChromeTabsApi } { + if (!defaultDeps) defaultDeps = { cdp: new ChromiumCdp(), tabsApi: chromeTabsApi }; + return defaultDeps; +} + +export async function handleWheel( + manager: SessionManager, + params: WheelParams, + deps: WheelDeps = getDefaultDeps(), +): Promise { + const ctxOrErr = lookupSession(manager, params, "wheel"); + if (isRpcError(ctxOrErr)) return ctxOrErr; + const ctx = ctxOrErr; + const deltaX = params.delta_x === undefined ? 0 : params.delta_x; + const deltaY = params.delta_y === undefined ? 0 : params.delta_y; + if (!Number.isFinite(deltaX) || !Number.isFinite(deltaY)) { + return { code: "invalid_params", message: "wheel deltas must be finite numbers" }; + } + if (deltaX === 0 && deltaY === 0) { + return { code: "invalid_params", message: "at least one wheel delta must be non-zero" }; + } + const timeout = params.timeout_ms === undefined ? 30_000 : params.timeout_ms; + if (!Number.isInteger(timeout) || timeout <= 0) { + return { code: "invalid_params", message: "timeout_ms must be a positive integer" }; + } + if ( + [params.ref, params.selector].some( + (value) => value !== undefined && (typeof value !== "string" || !value.trim()), + ) || + (params.ref !== undefined && params.selector !== undefined) + ) { + return { code: "invalid_params", message: "pass at most one nonempty ref or selector" }; + } + if ( + params.modifiers !== undefined && + (!Array.isArray(params.modifiers) || + params.modifiers.some((value) => !["alt", "ctrl", "meta", "shift"].includes(value))) + ) { + return { code: "invalid_params", message: "invalid wheel modifiers" }; + } + + const deadline = Date.now() + timeout; + const checkActive = () => { + if (deps.signal?.aborted) throw new DOMException("wheel aborted", "AbortError"); + if (Date.now() >= deadline) throw new DOMException("wheel timed out", "TimeoutError"); + }; + // Guard the shared target/geometry helpers without changing other tools. + // Own fallback scrolling and visibility objects, even if allocation is cancelled. + const objectGroup = `bsk-wheel-${crypto.randomUUID()}`; + const objectTargets = new Map(); + const send = async (target: CdpTarget, method: string, args?: object): Promise => { + checkActive(); + if (method === "DOM.resolveNode") objectTargets.set(cdpTargetKey(target), target); + const result = await sendToCdpTarget( + deps.cdp, + target, + method, + method === "DOM.resolveNode" ? { ...args, objectGroup } : args, + ); + checkActive(); + return result; + }; + let graph: ReturnType> | undefined; + const cdp: CdpRunner = { + send: (tabId, method, args) => send({ tabId }, method, args), + sendToTarget: send, + trackSessionTab: deps.cdp.trackSessionTab?.bind(deps.cdp), + getFrameGraph: deps.cdp.getFrameGraph + ? async (tabId) => { + checkActive(); + graph ??= deps.cdp.getFrameGraph!(tabId); + const result = await graph; + checkActive(); + return result; + } + : undefined, + }; + let bypassTab: number | undefined; + + try { + checkActive(); + const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi); + checkActive(); + if (isRpcError(target)) return target; + const denied = enforceAgentWindow(ctx, target, "wheel"); + if (denied) return denied; + const dialogCursor = markDialogCursor(deps.cdp, target.tabId); + cdp.trackSessionTab?.(ctx.sessionId, target.tabId); + const point = await resolveWheelPoint(cdp, ctx, target, params, deadline); + checkActive(); + if (isRpcError(point)) return point; + + if (deps.bypassOverlay) { + bypassTab = target.tabId; + await deps.bypassOverlay(target.tabId, true); + } + const modifiers = modifiersBitfield(params.modifiers); + await cdp.send(target.tabId, "Input.dispatchMouseEvent", { + type: "mouseMoved", + x: point.x, + y: point.y, + modifiers, + }); + await cdp.send(target.tabId, "Input.dispatchMouseEvent", { + type: "mouseWheel", + x: point.x, + y: point.y, + deltaX, + deltaY, + modifiers, + }); + return attachDialogs(deps.cdp, target.tabId, dialogCursor, { + tab_id: target.tabId, + used_ref: point.usedRef, + used_selector: point.usedSelector, + x: point.x, + y: point.y, + delta_x: deltaX, + delta_y: deltaY, + }); + } catch (error) { + if (deps.signal?.aborted) return { code: "cancelled", message: "wheel aborted" }; + if (Date.now() >= deadline) return { code: "timeout", message: "wheel timed out" }; + return cdpError(error); + } finally { + if (bypassTab !== undefined && deps.bypassOverlay) { + try { + await deps.bypassOverlay(bypassTab, false); + } catch (error) { + console.debug("[bsk wheel] overlay bypass disable failed", error); + } + } + for (const target of objectTargets.values()) { + await sendToCdpTarget(deps.cdp, target, "Runtime.releaseObjectGroup", { objectGroup }).catch( + () => {}, + ); + } + } +} + +async function resolveWheelPoint( + cdp: CdpRunner, + ctx: SessionContext, + target: { tabId: number }, + params: WheelParams, + deadline: number, +): Promise { + const hasRef = typeof params.ref === "string" && params.ref.length > 0; + const hasSelector = typeof params.selector === "string" && params.selector.length > 0; + if (hasRef || hasSelector) { + const node = await resolveBackendNode(cdp, ctx, target, params, "wheel"); + if (isRpcError(node)) return node; + const geometry = await resolveNodeGeometry( + cdp, + target.tabId, + { + target: node.cdpTarget, + backendNodeId: node.backendNodeId, + ...(node.frameId ? { frameId: node.frameId } : {}), + }, + { scrollIntoView: true }, + ); + if (isRpcError(geometry)) return geometry; + const bounds = await scrollVisibleBounds( + cdp, + target.tabId, + { target: node.cdpTarget, backendNodeId: node.backendNodeId, frameId: node.frameId }, + deadline, + ); + const clip = + bounds && rectPolygon({ x: bounds.x, y: bounds.y, w: bounds.width, h: bounds.height }); + const regions = clip + ? geometry.topVisibleRegions.map((region) => clipPolygon(region, clip)) + : []; + const visible = regions.sort((a, b) => polygonArea(b) - polygonArea(a))[0]; + const point = visible && polygonArea(visible) > 0 ? polygonCentroid(visible) : null; + if (!point) + return rpcError( + "permission_denied", + "element_not_visible", + "wheel target has no visible area", + ); + return { + x: point.x, + y: point.y, + usedRef: node.usedRef, + usedSelector: node.usedSelector, + }; + } + + const metrics = await cdp.send<{ + cssLayoutViewport?: { clientWidth?: number; clientHeight?: number }; + layoutViewport?: { clientWidth?: number; clientHeight?: number }; + }>(target.tabId, "Page.getLayoutMetrics", {}); + const viewport = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {}; + const width = viewport.clientWidth; + const height = viewport.clientHeight; + if ( + typeof width !== "number" || + typeof height !== "number" || + !Number.isFinite(width) || + !Number.isFinite(height) || + width <= 0 || + height <= 0 + ) { + return { code: "cdp_failed", message: "Page.getLayoutMetrics returned no viewport size" }; + } + return { x: width / 2, y: height / 2 }; +} diff --git a/apps/extension/src/transport/types.ts b/apps/extension/src/transport/types.ts index 5020b292..8196bd28 100644 --- a/apps/extension/src/transport/types.ts +++ b/apps/extension/src/transport/types.ts @@ -470,6 +470,28 @@ export interface HoverResult { dialogs?: JavaScriptDialogInfo[]; } +export interface WheelParams { + session_id: string; + ref?: string; + selector?: string; + tab_id?: number; + delta_x?: number; + delta_y?: number; + modifiers?: KeyModifier[]; + timeout_ms?: number; +} + +export interface WheelResult { + tab_id: number; + used_ref?: string; + used_selector?: string; + x: number; + y: number; + delta_x: number; + delta_y: number; + dialogs?: JavaScriptDialogInfo[]; +} + export interface ScrollToParams { session_id: string; ref?: string; diff --git a/crates/bsk-cli/skill/SKILL.md b/crates/bsk-cli/skill/SKILL.md index 765b91cc..3eb44796 100644 --- a/crates/bsk-cli/skill/SKILL.md +++ b/crates/bsk-cli/skill/SKILL.md @@ -55,7 +55,7 @@ Use this default loop: ```text bsk navigate --session bsk observe --session -bsk click|hover|scroll-to|focus|blur|fill|select|press ... --session +bsk click|hover|wheel|scroll-to|focus|blur|fill|select|press ... --session bsk observe --session # after navigation or a meaningful DOM change ``` @@ -66,6 +66,11 @@ ancestor clipping. Partial visibility is enough; hidden or fully clipped targets `permission_denied` and `data.reason=element_not_visible`. This does not test occlusion by other elements. For a specific tab or deadline: `bsk scroll-to @e3 --session --tab-id 42 --timeout 5s`. +`bsk wheel --delta-y -120 --session ` sends native wheel input at the viewport centre. +Add an optional ref/selector to target an element (scrolled into view first). Both delta axes +accept signed numbers and default to zero; at least one must be nonzero. The result echoes +input, not actual scroll distance or completion. Observe afterwards to check the page's response. + `bsk focus ` explicitly focuses a target; `bsk blur ` removes focus and reports whether it was focused. Use these for UI states triggered by focus changes. @@ -124,7 +129,7 @@ This list of names is complete. Never invent a command outside it; read session start|stop|list browsers status doctor update logs navigate navigate-back navigate-forward reload wait-for-navigation wait-ms observe snapshot get-html screenshot console network -click hover scroll-to focus blur fill select press evaluate +click hover wheel scroll-to focus blur fill select press evaluate tab list|create|close|select|borrow|return window resize emulate upload download request-help record start|stop ``` diff --git a/crates/bsk-cli/src/cli/mod.rs b/crates/bsk-cli/src/cli/mod.rs index 4d0f9d2e..2f3d20b1 100644 --- a/crates/bsk-cli/src/cli/mod.rs +++ b/crates/bsk-cli/src/cli/mod.rs @@ -36,6 +36,7 @@ pub mod tab; pub mod update; pub mod upload; pub mod waits; +pub mod wheel; pub mod window; use clap::{Args, Parser, Subcommand}; @@ -63,6 +64,7 @@ use crate::cli::tab::TabCmd; use crate::cli::update::UpdateArgs; use crate::cli::upload::UploadArgs; use crate::cli::waits::{WaitForNavigationArgs, WaitMsArgs}; +use crate::cli::wheel::WheelArgs; use crate::cli::window::WindowCmd; /// Tool calls wait slightly longer than the daemon's 30s tool timeout so @@ -177,6 +179,9 @@ pub enum Command { /// Hover a snapshot ref or CSS selector. Hover(HoverArgs), + /// Dispatch a native mouse-wheel event at the viewport centre or an element. + Wheel(WheelArgs), + /// Scroll a snapshot ref or CSS selector into the visible viewport. #[command(name = "scroll-to")] ScrollTo(ScrollToArgs), diff --git a/crates/bsk-cli/src/cli/wheel.rs b/crates/bsk-cli/src/cli/wheel.rs new file mode 100644 index 00000000..303bc747 --- /dev/null +++ b/crates/bsk-cli/src/cli/wheel.rs @@ -0,0 +1,213 @@ +//! `bsk wheel` — dispatch a native Chromium mouse-wheel event. + +use std::time::Duration; + +use anyhow::Context; +use bsk_protocol::Method; +use bsk_protocol::tools::{WheelParams, WheelResult}; +use clap::Args; + +use crate::cli::dialogs::print_dialog_summaries; +use crate::cli::ensure_daemon::ensure_daemon; +use crate::cli::error::{CliError, Format}; +use crate::cli::interaction::{parse_modifiers, split_target}; +use crate::cli::navigate::parse_timeout_ms; + +#[derive(Debug, Clone, Args)] +pub struct WheelArgs { + /// Optional snapshot ref (`@e3`, `e3`) or CSS selector used as the wheel hit-test point. + pub target: Option, + + #[arg(long = "ref")] + pub ref_: Option, + + #[arg(long = "selector")] + pub selector: Option, + + #[arg(long)] + pub session: String, + + #[arg(long = "tab-id")] + pub tab_id: Option, + + /// Horizontal wheel input in CSS pixels (positive is right). + #[arg(long = "delta-x", default_value = "0", allow_negative_numbers = true, value_parser = parse_finite_delta)] + pub delta_x: f64, + + /// Vertical wheel input in CSS pixels (positive is down). + #[arg(long = "delta-y", default_value = "0", allow_negative_numbers = true, value_parser = parse_finite_delta)] + pub delta_y: f64, + + /// Comma-separated modifiers (`alt,ctrl,shift,meta`). + #[arg(long, default_value = "")] + pub modifiers: String, + + #[arg(long, default_value = "30s", value_parser = parse_timeout_ms)] + pub timeout: u32, +} + +pub fn dispatch(args: WheelArgs, format: Format) -> Result<(), CliError> { + validate_deltas(args.delta_x, args.delta_y)?; + let (ref_, selector) = split_optional_target(args.target, args.ref_, args.selector)?; + let modifiers = parse_modifiers(&args.modifiers) + .map_err(|e| CliError::Local(anyhow::anyhow!("--modifiers: {e}")))?; + let info = ensure_daemon().context("ensure daemon is running")?; + let params = WheelParams { + session_id: args.session, + ref_, + selector, + tab_id: args.tab_id, + delta_x: args.delta_x, + delta_y: args.delta_y, + modifiers: if modifiers.is_empty() { + None + } else { + Some(modifiers) + }, + timeout_ms: Some(args.timeout), + }; + let reply: WheelResult = crate::cli::business_rpc::call( + info.sock_path, + "wheel-1", + Method::ToolWheel, + Some(params), + ipc_timeout(args.timeout), + )?; + match format { + Format::Json => println!( + "{}", + serde_json::to_string_pretty(&reply) + .map_err(|e| CliError::Local(anyhow::anyhow!(e)))? + ), + Format::Human => { + let target = + format_used_target(reply.used_ref.as_deref(), reply.used_selector.as_deref()); + println!( + "wheel ok tab={} target={target} at=({}, {}) delta=({}, {})", + reply.tab_id, reply.x, reply.y, reply.delta_x, reply.delta_y + ); + print_dialog_summaries(&reply.dialogs); + } + } + Ok(()) +} + +fn parse_finite_delta(value: &str) -> Result { + let parsed = value + .parse::() + .map_err(|_| format!("'{value}' is not a valid number"))?; + if !parsed.is_finite() { + return Err("wheel deltas must be finite numbers".into()); + } + Ok(parsed) +} + +fn validate_deltas(delta_x: f64, delta_y: f64) -> Result<(), CliError> { + if !delta_x.is_finite() || !delta_y.is_finite() { + return Err(CliError::Local(anyhow::anyhow!( + "wheel deltas must be finite numbers" + ))); + } + if delta_x == 0.0 && delta_y == 0.0 { + return Err(CliError::Local(anyhow::anyhow!( + "at least one of --delta-x or --delta-y must be non-zero" + ))); + } + Ok(()) +} + +fn split_optional_target( + positional: Option, + explicit_ref: Option, + explicit_selector: Option, +) -> Result<(Option, Option), CliError> { + if [&positional, &explicit_ref, &explicit_selector] + .iter() + .any(|target| target.as_ref().is_some_and(|value| value.trim().is_empty())) + { + return Err(CliError::Local(anyhow::anyhow!( + "wheel target must not be empty" + ))); + } + if positional.is_none() && explicit_ref.is_none() && explicit_selector.is_none() { + Ok((None, None)) + } else { + split_target(positional, explicit_ref, explicit_selector) + } +} + +fn format_used_target(used_ref: Option<&str>, used_selector: Option<&str>) -> String { + used_ref + .map(|r| format!("@{r}")) + .or_else(|| used_selector.map(str::to_string)) + .unwrap_or_else(|| "viewport-center".into()) +} + +fn ipc_timeout(timeout_ms: u32) -> Duration { + Duration::from_millis(u64::from(timeout_ms)) + .checked_add(Duration::from_secs(15)) + .unwrap_or(Duration::from_secs(u64::from(timeout_ms / 1_000) + 15)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn optional_target_supports_viewport_refs_and_selectors() { + assert_eq!( + split_optional_target(None, None, None).unwrap(), + (None, None) + ); + assert_eq!( + split_optional_target(Some("@e3".into()), None, None).unwrap(), + (Some("@e3".into()), None) + ); + assert_eq!( + split_optional_target(Some("#panel".into()), None, None).unwrap(), + (None, Some("#panel".into())) + ); + } + + #[test] + fn finite_nonzero_deltas_are_required() { + assert!(parse_finite_delta("600").is_ok()); + assert!(parse_finite_delta("NaN").is_err()); + assert!(validate_deltas(0.0, 0.0).is_err()); + assert!(validate_deltas(0.0, -120.0).is_ok()); + } + #[test] + fn rejects_empty_conflicting_targets_and_nonfinite_deltas() { + for target in ["", " "] { + assert!(split_optional_target(Some(target.into()), None, None).is_err()); + assert!(split_optional_target(None, Some(target.into()), None).is_err()); + assert!(split_optional_target(None, None, Some(target.into())).is_err()); + } + assert!(split_optional_target(Some("e3".into()), Some("e4".into()), None).is_err()); + assert!(split_optional_target(None, Some("e3".into()), Some("#panel".into())).is_err()); + assert!(validate_deltas(f64::INFINITY, 120.0).is_err()); + assert!(validate_deltas(0.0, f64::NAN).is_err()); + } + + #[test] + fn rejects_invalid_commands_before_starting_the_daemon() { + use crate::cli::{Cli, Command}; + use clap::Parser; + for options in [ + vec![], + vec!["--delta-y", "120", "--ref", ""], + vec!["--delta-y", "120", "--ref", "e3", "--selector", "#panel"], + vec!["--delta-y", "120", "--modifiers", "invalid"], + ] { + let mut argv = vec!["bsk", "wheel", "--session", "s1"]; + argv.extend(options); + let Command::Wheel(args) = Cli::try_parse_from(argv).unwrap().command else { + panic!("wheel"); + }; + assert!(matches!( + dispatch(args, Format::Json), + Err(CliError::Local(_)) + )); + } + } +} diff --git a/crates/bsk-cli/src/daemon/ipc.rs b/crates/bsk-cli/src/daemon/ipc.rs index 440a7f50..42696692 100644 --- a/crates/bsk-cli/src/daemon/ipc.rs +++ b/crates/bsk-cli/src/daemon/ipc.rs @@ -253,6 +253,7 @@ pub fn full_handler(status: DaemonStatus, state: Arc) -> RpcHandler | Method::ToolReload | Method::ToolClick | Method::ToolHover + | Method::ToolWheel | Method::ToolScrollTo | Method::ToolFocus | Method::ToolBlur diff --git a/crates/bsk-cli/src/daemon/queue.rs b/crates/bsk-cli/src/daemon/queue.rs index 6b7366a4..68634bf1 100644 --- a/crates/bsk-cli/src/daemon/queue.rs +++ b/crates/bsk-cli/src/daemon/queue.rs @@ -438,7 +438,7 @@ async fn dispatch_with_sender( lifecycle_cancel: Option, cancel_cleanup_timeout: Duration, ) -> Result { - let effect_aware_transfer = is_effect_aware_transfer(&method); + let deadline_cleanup = waits_for_deadline_cleanup(&method); let (respond_tx, respond_rx) = oneshot::channel(); let job = ToolJob { method, @@ -475,10 +475,10 @@ async fn dispatch_with_sender( mpsc::error::TrySendError::Closed(_) => DispatchError::QueueClosed, }); } - // Lifecycle teardown and effect-aware transfers both keep the worker busy + // Lifecycle teardown, wheel input and effect-aware transfers keep the worker busy // for bounded compensation after their original deadline. Keep the outer // waiter alive for the same grace period so it cannot abandon reconciliation. - let response_grace = if lifecycle_cancellable || effect_aware_transfer { + let response_grace = if lifecycle_cancellable || deadline_cleanup { cancel_cleanup_timeout } else { Duration::ZERO @@ -654,7 +654,7 @@ async fn forward_one( .is_ok() }; let deadline_cancel: Option<&(dyn Fn() -> bool + Sync)> = - is_effect_aware_transfer(&job.method).then_some(&send_deadline_cancel); + waits_for_deadline_cleanup(&job.method).then_some(&send_deadline_cancel); let waited = await_with_optional_cancel( job.timeout, job.cancel_cleanup_timeout, @@ -761,6 +761,13 @@ async fn forward_one( } WaitOutcome::TimeoutCleanupFailed => { client.pending.lock().unwrap().cancel(&rpc_id); + if job.method == Method::ToolWheel { + return Err(RpcError { + code: ErrorCode::Timeout, + message: "wheel timed out and extension cleanup could not be confirmed".into(), + data: Some(serde_json::json!({ "reason": "cancel_cleanup_timeout" })), + }); + } return Err(unknown_transfer_error( ErrorCode::Timeout, "file transfer timed out and cleanup could not be confirmed", @@ -807,6 +814,12 @@ async fn forward_one( } } +// Wheel must also cancel at the daemon deadline: its extension-local deadline +// cannot account for a request delayed in transit. Other input tools are unchanged. +fn waits_for_deadline_cleanup(method: &Method) -> bool { + *method == Method::ToolWheel || is_effect_aware_transfer(method) +} + fn is_effect_aware_transfer(method: &Method) -> bool { matches!(method, Method::ToolUpload | Method::ToolDownload) } diff --git a/crates/bsk-cli/src/main.rs b/crates/bsk-cli/src/main.rs index 19aff7e5..1fd447e4 100644 --- a/crates/bsk-cli/src/main.rs +++ b/crates/bsk-cli/src/main.rs @@ -94,6 +94,7 @@ fn dispatch(cli: Cli, format: Format) -> Result<(), CliError> { Command::Reload(args) => cli::navigate::dispatch_reload(args, format), Command::Click(args) => cli::interaction::dispatch_click(args, format), Command::Hover(args) => cli::interaction::dispatch_hover(args, format), + Command::Wheel(args) => cli::wheel::dispatch(args, format), Command::ScrollTo(args) => cli::scroll::dispatch(args, format), Command::Focus(args) => cli::interaction::dispatch_focus(args, format), Command::Blur(args) => cli::interaction::dispatch_blur(args, format), diff --git a/crates/bsk-cli/tests/cli_parse.rs b/crates/bsk-cli/tests/cli_parse.rs index 11666e4c..f94f545c 100644 --- a/crates/bsk-cli/tests/cli_parse.rs +++ b/crates/bsk-cli/tests/cli_parse.rs @@ -255,6 +255,27 @@ fn parses_hover_with_settle() { assert_eq!(args.settle, 300); } +#[test] +fn parses_wheel_with_target_and_deltas() { + let cli = parse(&[ + "bsk", + "wheel", + "#panel", + "--delta-x", + "12.5", + "--delta-y", + "600", + "--session", + "s1", + ]); + let Command::Wheel(args) = cli.command else { + panic!("expected wheel command"); + }; + assert_eq!(args.target.as_deref(), Some("#panel")); + assert_eq!(args.delta_x, 12.5); + assert_eq!(args.delta_y, 600.0); +} + #[test] fn parses_scroll_to_target() { let cli = parse(&["bsk", "scroll-to", "@e2", "--session", "s1"]); @@ -660,3 +681,37 @@ fn parses_session_start_no_focus() { }; assert!(args.no_focus); } + +#[test] +fn parses_signed_wheel_deltas_and_optional_axes() { + for (options, expected) in [ + (vec!["--delta-y", "-120"], (0.0, -120.0)), + (vec!["--delta-x", "-20.5"], (-20.5, 0.0)), + (vec!["--delta-y=-120"], (0.0, -120.0)), + ( + vec!["--delta-x", "12.5", "--delta-y", "-600"], + (12.5, -600.0), + ), + ] { + let mut argv = vec!["bsk", "wheel", "--session", "s1"]; + argv.extend(options); + let Command::Wheel(args) = parse(&argv).command else { + panic!("expected wheel"); + }; + assert_eq!((args.delta_x, args.delta_y), expected); + } +} + +#[test] +fn rejects_invalid_wheel_numbers_and_timeouts() { + for options in [ + vec!["--delta-y", "NaN"], + vec!["--delta-x", "inf"], + vec!["--delta-y", "oops"], + vec!["--delta-y", "120", "--timeout", "0ms"], + ] { + let mut argv = vec!["bsk", "wheel", "--session", "s1"]; + argv.extend(options); + assert!(Cli::try_parse_from(argv).is_err()); + } +} diff --git a/crates/bsk-cli/tests/per_session_queue.rs b/crates/bsk-cli/tests/per_session_queue.rs index 70df3a7d..b502e49f 100644 --- a/crates/bsk-cli/tests/per_session_queue.rs +++ b/crates/bsk-cli/tests/per_session_queue.rs @@ -182,13 +182,19 @@ async fn run_fake_extension( .unwrap(); } _ => { - let tag = req - .params - .as_ref() - .and_then(|p| p.get("tag")) - .and_then(|v| v.as_str()) - .unwrap_or_default() - .to_string(); + let tag = if req.method == Method::Cancel { + format!( + "cancel:{}", + req.params.as_ref().unwrap()["rpc_id"].as_str().unwrap() + ) + } else { + req.params + .as_ref() + .and_then(|p| p.get("tag")) + .and_then(|v| v.as_str()) + .unwrap_or_default() + .to_string() + }; let _ = requests_tx.send((req.id.clone(), tag)); } } @@ -578,3 +584,100 @@ async fn dispatch_returns_session_not_found_for_unknown_session() { assert_eq!(rpc.code, ErrorCode::NotFound); handle.shutdown().await; } + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn wheel_deadline_cancels_the_extension_and_keeps_the_session_busy_during_cleanup() { + for acknowledge_cleanup in [true, false] { + let (handle, sock) = spawn_daemon().await; + let mut ws = connect_ext(handle.ws_addr()).await; + let _ = handshake_as_ext(&mut ws).await; + let (req_tx, mut req_rx) = mpsc::unbounded_channel(); + let (reply_tx, reply_rx) = mpsc::unbounded_channel(); + tokio::spawn(run_fake_extension( + ws, + Arc::new(Mutex::new(100)), + req_tx, + reply_rx, + )); + let session_id = ipc_session_start(&sock).await; + let sid = bsk::daemon::sessions::SessionId(session_id); + let queues = Arc::clone(&handle.state().tool_queues); + let task = { + let queues = Arc::clone(&queues); + let sid = sid.clone(); + tokio::spawn(async move { + queues + .dispatch( + &sid, + Method::ToolWheel, + json!({"session_id":sid.0,"delta_y":120}), + Duration::from_millis(20), + None, + ) + .await + }) + }; + let (rpc_id, _) = tokio::time::timeout(Duration::from_secs(2), req_rx.recv()) + .await + .unwrap() + .unwrap(); + let (cancel_id, tag) = tokio::time::timeout(Duration::from_secs(2), req_rx.recv()) + .await + .unwrap() + .unwrap(); + assert_eq!(cancel_id, format!("deadline-cancel-{rpc_id}")); + assert_eq!(tag, format!("cancel:{rpc_id}")); + // Outlive the ordinary response grace: the caller and queue must still + // wait for wheel cleanup instead of abandoning the outstanding input. + tokio::time::sleep(Duration::from_millis(1100)).await; + assert!(!task.is_finished()); + assert!(matches!( + queues + .dispatch( + &sid, + Method::ToolTabList, + json!({"session_id":sid.0}), + Duration::from_secs(1), + None + ) + .await, + Err(DispatchError::SessionBusy) + )); + if acknowledge_cleanup { + // Even a late success cannot turn a daemon deadline into success. + reply_tx.send((rpc_id, json!({"delta_y":120}))).unwrap(); + } + let Err(DispatchError::Rpc(error)) = tokio::time::timeout(Duration::from_secs(3), task) + .await + .unwrap() + .unwrap() + else { + panic!("expected a structured timeout"); + }; + assert_eq!(error.code, ErrorCode::Timeout); + if !acknowledge_cleanup { + assert_eq!(error.data.unwrap()["reason"], "cancel_cleanup_timeout"); + } + let next = { + let sid = sid.clone(); + tokio::spawn(async move { + queues + .dispatch( + &sid, + Method::ToolTabList, + json!({"session_id":sid.0}), + Duration::from_secs(2), + None, + ) + .await + }) + }; + let (next_id, _) = tokio::time::timeout(Duration::from_secs(2), req_rx.recv()) + .await + .unwrap() + .unwrap(); + reply_tx.send((next_id, json!({"ok":true}))).unwrap(); + assert!(next.await.unwrap().is_ok()); + handle.shutdown().await; + } +} diff --git a/crates/bsk-cli/tests/session_user_interrupt.rs b/crates/bsk-cli/tests/session_user_interrupt.rs index 5d3cbd36..2eca043f 100644 --- a/crates/bsk-cli/tests/session_user_interrupt.rs +++ b/crates/bsk-cli/tests/session_user_interrupt.rs @@ -301,6 +301,11 @@ async fn user_interrupt_rejects_scroll_to_before_forwarding() { assert_idle_interrupt_rejects(Method::ToolScrollTo).await; } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn user_interrupt_rejects_wheel_before_forwarding() { + assert_idle_interrupt_rejects(Method::ToolWheel).await; +} + async fn assert_idle_interrupt_rejects(method: Method) { let (handle, sock) = spawn_daemon().await; let mut ws = connect_ext(handle.ws_addr()).await; diff --git a/crates/bsk-cli/tests/tools_m7_ipc.rs b/crates/bsk-cli/tests/tools_m7_ipc.rs index e8c2c42c..1a1b46f8 100644 --- a/crates/bsk-cli/tests/tools_m7_ipc.rs +++ b/crates/bsk-cli/tests/tools_m7_ipc.rs @@ -16,7 +16,7 @@ use bsk_protocol::tools::{ FocusResult, KeyModifier, MouseButton, NavigateBackParams, NavigateBackResult, NavigateForwardParams, NavigateForwardResult, NavigateParams, NavigateResult, PressParams, PressResult, ReloadParams, ReloadResult, SelectParams, SelectResult, SessionStartParams, - SessionStartResult, WaitUntil, + SessionStartResult, WaitUntil, WheelParams, WheelResult, }; use bsk_protocol::tools::{ScrollToParams, ScrollToResult}; use bsk_protocol::{ @@ -801,3 +801,52 @@ async fn m7_tools_propagate_extension_errors() { handle.shutdown().await; } + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn wheel_round_trips_native_deltas_and_point() { + let (handle, sock) = spawn_daemon().await; + let mut ws = connect_ext(handle.ws_addr()).await; + let _ = do_handshake(&mut ws).await; + + run_extension(ws, |req| { + assert_eq!(req.method, Method::ToolWheel); + let params: WheelParams = serde_json::from_value(req.params.clone().unwrap()).unwrap(); + assert_eq!(params.selector.as_deref(), Some("#panel")); + assert_eq!(params.delta_x, 10.0); + assert_eq!(params.delta_y, 600.0); + ResponseBody::Ok( + serde_json::to_value(WheelResult { + tab_id: 9, + used_ref: None, + used_selector: Some("#panel".into()), + x: 320.0, + y: 240.0, + delta_x: 10.0, + delta_y: 600.0, + dialogs: vec![], + }) + .unwrap(), + ) + }); + + let session_id = ipc_session_start(&sock).await; + let result: WheelResult = ipc_tool_call( + &sock, + Method::ToolWheel, + WheelParams { + session_id, + ref_: None, + selector: Some("#panel".into()), + tab_id: None, + delta_x: 10.0, + delta_y: 600.0, + modifiers: None, + timeout_ms: Some(5_000), + }, + ) + .await + .expect("wheel ok"); + assert_eq!(result.used_selector.as_deref(), Some("#panel")); + assert_eq!((result.delta_x, result.delta_y), (10.0, 600.0)); + handle.shutdown().await; +} diff --git a/crates/bsk-protocol/schema/tool_wheel_params.json b/crates/bsk-protocol/schema/tool_wheel_params.json new file mode 100644 index 00000000..c5b40403 --- /dev/null +++ b/crates/bsk-protocol/schema/tool_wheel_params.json @@ -0,0 +1,75 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "WheelParams", + "type": "object", + "required": [ + "session_id" + ], + "properties": { + "delta_x": { + "description": "Native CDP wheel input in CSS pixels. Positive X means right. Not measured displacement.", + "default": 0.0, + "type": "number", + "format": "double" + }, + "delta_y": { + "description": "Native CDP wheel input in CSS pixels. Positive Y means down. Defaults to zero.", + "default": 0.0, + "type": "number", + "format": "double" + }, + "modifiers": { + "type": [ + "array", + "null" + ], + "items": { + "$ref": "#/definitions/KeyModifier" + } + }, + "ref": { + "description": "Optional `@e` ref used as the wheel event's hit-test point. Mutually exclusive with `selector`. When neither is supplied, the event is dispatched at the viewport centre.", + "type": [ + "string", + "null" + ] + }, + "selector": { + "type": [ + "string", + "null" + ] + }, + "session_id": { + "type": "string" + }, + "tab_id": { + "description": "Target tab. Defaults to the Agent Window's active tab.", + "type": [ + "integer", + "null" + ], + "format": "int64" + }, + "timeout_ms": { + "type": [ + "integer", + "null" + ], + "format": "uint32", + "minimum": 1.0 + } + }, + "definitions": { + "KeyModifier": { + "description": "Keyboard modifier flags. Multiple flags may be combined; the extension folds them into CDP's bitfield (`alt=1, ctrl=2, meta=4, shift=8`).", + "type": "string", + "enum": [ + "alt", + "ctrl", + "meta", + "shift" + ] + } + } +} diff --git a/crates/bsk-protocol/schema/tool_wheel_result.json b/crates/bsk-protocol/schema/tool_wheel_result.json new file mode 100644 index 00000000..e81785bf --- /dev/null +++ b/crates/bsk-protocol/schema/tool_wheel_result.json @@ -0,0 +1,124 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "WheelResult", + "type": "object", + "required": [ + "delta_x", + "delta_y", + "tab_id", + "x", + "y" + ], + "properties": { + "delta_x": { + "description": "Requested deltas, not measured scroll distance or completion.", + "type": "number", + "format": "double" + }, + "delta_y": { + "type": "number", + "format": "double" + }, + "dialogs": { + "type": "array", + "items": { + "$ref": "#/definitions/JavaScriptDialogInfo" + } + }, + "tab_id": { + "type": "integer", + "format": "int64" + }, + "used_ref": { + "type": [ + "string", + "null" + ] + }, + "used_selector": { + "type": [ + "string", + "null" + ] + }, + "x": { + "description": "Top-level viewport coordinates where the real wheel event was dispatched.", + "type": "number", + "format": "double" + }, + "y": { + "type": "number", + "format": "double" + } + }, + "definitions": { + "JavaScriptDialogHandledAction": { + "description": "How the extension resolved the dialog so CDP could continue.", + "type": "string", + "enum": [ + "accepted", + "dismissed" + ] + }, + "JavaScriptDialogInfo": { + "description": "One observed + handled JavaScript dialog during a tool call.", + "type": "object", + "required": [ + "handled", + "message", + "sequence", + "tab_id", + "type" + ], + "properties": { + "default_prompt": { + "type": [ + "string", + "null" + ] + }, + "handled": { + "$ref": "#/definitions/JavaScriptDialogHandledAction" + }, + "has_browser_handler": { + "type": [ + "boolean", + "null" + ] + }, + "message": { + "type": "string" + }, + "sequence": { + "description": "Monotonic per-tab sequence for ordering within a session.", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "tab_id": { + "type": "integer", + "format": "int64" + }, + "type": { + "$ref": "#/definitions/JavaScriptDialogType" + }, + "url": { + "type": [ + "string", + "null" + ] + } + } + }, + "JavaScriptDialogType": { + "description": "Native JS dialog kind reported by CDP.", + "type": "string", + "enum": [ + "alert", + "confirm", + "prompt", + "beforeunload" + ] + } + } +} diff --git a/crates/bsk-protocol/src/bin/dump-schema.rs b/crates/bsk-protocol/src/bin/dump-schema.rs index fc60243a..7a156176 100644 --- a/crates/bsk-protocol/src/bin/dump-schema.rs +++ b/crates/bsk-protocol/src/bin/dump-schema.rs @@ -79,6 +79,8 @@ fn main() { dump!(ClickResult, "tool_click_result"); dump!(HoverParams, "tool_hover_params"); dump!(HoverResult, "tool_hover_result"); + dump!(WheelParams, "tool_wheel_params"); + dump!(WheelResult, "tool_wheel_result"); dump!(ScrollToParams, "tool_scroll_to_params"); dump!(ScrollToResult, "tool_scroll_to_result"); dump!(FocusParams, "tool_focus_params"); diff --git a/crates/bsk-protocol/src/method.rs b/crates/bsk-protocol/src/method.rs index 4fb4cf6d..b773847e 100644 --- a/crates/bsk-protocol/src/method.rs +++ b/crates/bsk-protocol/src/method.rs @@ -72,6 +72,8 @@ pub enum Method { ToolClick, #[serde(rename = "tool.hover")] ToolHover, + #[serde(rename = "tool.wheel")] + ToolWheel, #[serde(rename = "tool.scroll_to")] ToolScrollTo, #[serde(rename = "tool.focus")] @@ -174,6 +176,7 @@ impl Method { | Method::ToolNavigateForward | Method::ToolReload | Method::ToolClick + | Method::ToolWheel | Method::ToolScrollTo | Method::ToolFocus | Method::ToolBlur @@ -314,6 +317,7 @@ mod tests { assert!(Method::ToolNavigateForward.is_mutating()); assert!(Method::ToolReload.is_mutating()); assert!(Method::ToolClick.is_mutating()); + assert!(Method::ToolWheel.is_mutating()); assert!(Method::ToolScrollTo.is_mutating()); assert!(Method::ToolFocus.is_mutating()); assert!(Method::ToolBlur.is_mutating()); @@ -361,6 +365,7 @@ mod tests { assert_eq!(Method::ToolHover.effect(), MethodEffect::TransientInput); assert_eq!(Method::ToolObserve.effect(), MethodEffect::TransientInput); assert_eq!(Method::ToolClick.effect(), MethodEffect::BrowserMutation); + assert_eq!(Method::ToolWheel.effect(), MethodEffect::BrowserMutation); assert_eq!(Method::ToolScrollTo.effect(), MethodEffect::BrowserMutation); assert_eq!(Method::ToolFocus.effect(), MethodEffect::BrowserMutation); assert_eq!(Method::ToolBlur.effect(), MethodEffect::BrowserMutation); @@ -373,6 +378,7 @@ mod tests { assert!(Method::ToolHover.requires_interrupt_gate()); assert!(Method::ToolObserve.requires_interrupt_gate()); assert!(Method::ToolClick.requires_interrupt_gate()); + assert!(Method::ToolWheel.requires_interrupt_gate()); assert!(Method::ToolScrollTo.requires_interrupt_gate()); assert!(Method::ToolFocus.requires_interrupt_gate()); assert!(Method::ToolBlur.requires_interrupt_gate()); diff --git a/crates/bsk-protocol/src/tools/mod.rs b/crates/bsk-protocol/src/tools/mod.rs index 035f3289..72f6e285 100644 --- a/crates/bsk-protocol/src/tools/mod.rs +++ b/crates/bsk-protocol/src/tools/mod.rs @@ -18,6 +18,7 @@ pub mod scroll; pub mod session; pub mod tabs; pub mod waits; +pub mod wheel; pub mod window; pub use console::*; @@ -35,4 +36,5 @@ pub use scroll::*; pub use session::*; pub use tabs::*; pub use waits::*; +pub use wheel::*; pub use window::*; diff --git a/crates/bsk-protocol/src/tools/wheel.rs b/crates/bsk-protocol/src/tools/wheel.rs new file mode 100644 index 00000000..50d16979 --- /dev/null +++ b/crates/bsk-protocol/src/tools/wheel.rs @@ -0,0 +1,83 @@ +//! Native mouse-wheel input primitive (`tool.wheel`). + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +use super::{JavaScriptDialogInfo, KeyModifier}; + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct WheelParams { + pub session_id: String, + /// Optional `@e` ref used as the wheel event's hit-test point. + /// Mutually exclusive with `selector`. When neither is supplied, + /// the event is dispatched at the viewport centre. + #[serde( + rename = "ref", + alias = "ref_", + default, + skip_serializing_if = "Option::is_none" + )] + pub ref_: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub selector: Option, + /// Target tab. Defaults to the Agent Window's active tab. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tab_id: Option, + /// Native CDP wheel input in CSS pixels. Positive X means right. Not measured displacement. + #[serde(default)] + pub delta_x: f64, + /// Native CDP wheel input in CSS pixels. Positive Y means down. Defaults to zero. + #[serde(default)] + pub delta_y: f64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub modifiers: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(range(min = 1))] + pub timeout_ms: Option, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct WheelResult { + pub tab_id: i64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub used_ref: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub used_selector: Option, + /// Top-level viewport coordinates where the real wheel event was dispatched. + pub x: f64, + pub y: f64, + /// Requested deltas, not measured scroll distance or completion. + pub delta_x: f64, + pub delta_y: f64, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub dialogs: Vec, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn params_accept_missing_horizontal_delta_and_serialise_ref() { + let value = serde_json::json!({ + "session_id": "abcd", + "ref": "@e3", + "delta_y": 600.0 + }); + let params: WheelParams = serde_json::from_value(value).unwrap(); + assert_eq!(params.delta_x, 0.0); + assert_eq!(params.ref_.as_deref(), Some("@e3")); + + let round = serde_json::to_value(params).unwrap(); + assert_eq!(round.get("ref").and_then(|v| v.as_str()), Some("@e3")); + assert!(round.get("ref_").is_none()); + } + #[test] + fn params_accept_horizontal_only_input() { + let params: WheelParams = serde_json::from_value(serde_json::json!({ + "session_id": "abcd", "delta_x": -120.5 + })) + .unwrap(); + assert_eq!((params.delta_x, params.delta_y), (-120.5, 0.0)); + } +} diff --git a/docs/scroll-to.md b/docs/scroll-to.md index f1b3838b..d8176d18 100644 --- a/docs/scroll-to.md +++ b/docs/scroll-to.md @@ -3,7 +3,7 @@ `scroll-to` brings an existing element and its containing frames into view, then returns the visible portion's bounds. Use it to reveal a target before inspecting the page or taking a screenshot. It accepts an element target rather than a pixel -distance or scroll direction. +distance or scroll direction. Use [wheel](wheel.md) for native directional wheel input. | Entry point | Name | | --- | --- | diff --git a/docs/wheel.md b/docs/wheel.md new file mode 100644 index 00000000..2de3d26f --- /dev/null +++ b/docs/wheel.md @@ -0,0 +1,139 @@ +# Native wheel input + +`wheel` sends a native Chromium mouse-wheel event. Use it for directional +scrolling or controls that respond to wheel input. Use [scroll-to](scroll-to.md) +when the goal is simply to bring an existing element into view. + +| Entry point | Name | +| --- | --- | +| CLI | `bsk wheel` | +| Wire protocol | `tool.wheel` | +| DeepSeek Harness | `browser_interact` with `action: "wheel"` | + +## Usage + +Use a session you own and fresh refs from an observation. The ids below are examples. + +```sh +bsk wheel --delta-y 600 --session abcd +bsk wheel --delta-y -120 --session abcd +bsk wheel '#panel' --delta-x -200 --session abcd --tab-id 42 --timeout 5s --json +bsk wheel @e3 --delta-y 120 --modifiers ctrl,shift --session abcd +``` + +Both deltas default to zero. Values must be finite numbers, and at least one +must be nonzero. Positive X means right and positive Y means down. Negative +values work with either `--delta-y -120` or `--delta-y=-120`. + +Supply at most one target: a positional ref/selector, `--ref`, or `--selector`. +An explicitly empty target is an error. Refs accept `@e3` and `e3` and belong to +one session and tab. CSS selectors search the main document; use refs for +iframe or shadow-root elements. Unknown, stale and cross-tab refs fail instead +of falling back to the viewport. + +`--session` is required. `--tab-id` defaults to the Agent Window's active tab; +user tabs must first be borrowed into that window. `--timeout` defaults to +`30s` and must be positive. Modifiers are comma-separated `alt,ctrl,meta,shift`. + +## Input and result contract + +Without a target, the mouse moves to the viewport centre before the wheel event. +The browser hit-tests that point: an inner scroller under the centre may receive +the input instead of the document. + +With a target, the tool first scrolls the element and its frame owners into view. +It then picks a point within the element's visible content, accounting for +ancestor clipping and iframe projection, and moves the mouse there before +sending the wheel event. Hidden or fully clipped targets fail. Other overlapping +page elements can still receive the input; visibility is not an occlusion test. +Moving the pointer can itself trigger hover UI and change the page. + +Deltas are passed unchanged to CDP's `Input.dispatchMouseEvent`, whose input +units are CSS pixels. Chromium applies native page-zoom and modifier behavior; +DOM `WheelEvent.deltaX/deltaY` and actual scroll distance can differ from the +requested values. For example, Chromium at 125% page zoom delivers a 120-unit +input as a 96-unit DOM wheel delta. A page can also prevent the default scroll +or use the event to zoom a control. Browser behavior is preserved. + +Example result: + +```json +{ + "tab_id": 42, + "used_selector": "#panel", + "x": 320, + "y": 240, + "delta_x": -200, + "delta_y": 0 +} +``` + +`x/y` are the dispatch point in top-level viewport CSS pixels, not screenshot +pixels. `delta_x/delta_y` echo the requested input, not a measured displacement. +Success means input dispatch completed; it does not wait for scrolling, +animations or application work to finish. Observe the page afterwards to check +the result. The optional `dialogs` array reports dialogs handled during the call. + +## Protocol and plugin + +```json +{ + "id": "wheel-1", + "method": "tool.wheel", + "params": { + "session_id": "abcd", + "ref": "@e3", + "delta_y": -120, + "timeout_ms": 5000 + } +} +``` + +Both axes are optional and default to zero; the handler enforces nonzero input, +finite deltas and at most one nonempty `ref` or `selector`. See the generated +[parameter schema](../crates/bsk-protocol/schema/tool_wheel_params.json) and +[result schema](../crates/bsk-protocol/schema/tool_wheel_result.json). + +The plugin exposes the same operation through a plugin-owned session: + +```json +{ + "action": "wheel", + "session": "abcd", + "target": "@e3", + "deltaY": -120, + "tabId": 42, + "timeoutMs": 5000 +} +``` + +Omitted `session` uses the current plugin session. The result contains `session`, +`tabId`, `x`, `y`, `deltaX` and `deltaY`, with the same input semantics above. +The action uses the existing ownership, cancellation, queue and observation path. + +## Errors and interruption + +Invalid input returns `invalid_params`; missing refs/selectors return `not_found`. +An out-of-scope tab or invisible target returns `permission_denied`. Geometry, +visibility, overlay bypass and input failures return `cdp_failed`. + +Wheel is a browser mutation and uses the session queue and user-interruption +gate. On timeout, the daemon sends cancellation to the extension and keeps the +session busy while waiting for cleanup (bounded to two seconds). This also covers +requests delayed in transit. Cancellation and deadline checks surround browser steps, including tab +resolution, selector lookup, frame scrolling, measurement and input. Once +observed, cancellation or timeout prevents further input and is reported as +`cancelled` or `timeout`. Overlay restoration and remote-object cleanup still run. +Already performed scrolling or dispatched input cannot be recalled; inspect the +current page before retrying an interrupted call. + +## Browser regression tests + +The opt-in suite creates an isolated headless Chromium profile and local fixture +servers. It tests the handler through real CDP, including trusted events, +scrolling, nested clips, same-process and out-of-process iframes, zoom, modifiers, +`preventDefault` and cancellation. It does not attach to a user's browser. + +```sh +BSK_WHEEL_CHROME=/path/to/chrome pnpm --filter @browser-skill/extension exec vitest run src/tools/__tests__/wheel.browser.test.ts +``` diff --git a/packages/dsh-plugin-browserskill/README.md b/packages/dsh-plugin-browserskill/README.md index b2d13bf6..9e47a2f4 100644 --- a/packages/dsh-plugin-browserskill/README.md +++ b/packages/dsh-plugin-browserskill/README.md @@ -54,10 +54,13 @@ the `bsk` CLI and browser extension separately when a release requires it. | `browser_session` | `start`, `stop`, `list` | Manage plugin-owned Agent Window sessions. | | `browser_page` | `navigate`, `back`, `forward`, `reload`, `wait` | Navigate the active tab and wait for page lifecycle events. | | `browser_inspect` | `observe`, `snapshot`, `html`, `screenshot`, `console`, `network` | Read semantic or diagnostic page state and capture screenshots. | -| `browser_interact` | `click`, `hover`, `scroll-to`, `focus`, `blur`, `fill`, `select`, `press` | Interact with controls using fresh refs or selectors. | +| `browser_interact` | `click`, `hover`, `wheel`, `scroll-to`, `focus`, `blur`, `fill`, `select`, `press` | Interact with controls using fresh refs or selectors. | | `browser_tabs` | `list`, `create`, `select`, `close`, `borrow`, `return` | Manage Agent Window tabs and temporarily borrow user tabs. | | `browser_assist` | `resize`, `emulate`, `request-help` | Resize or emulate the browser and pause for human-only steps. | +For native wheel input (`action: "wheel"`), see the [wheel reference](../../docs/wheel.md) +for signed deltas, optional targets, result semantics and interruption. + For `browser_interact` with `action: "scroll-to"`, see the [scroll-to reference](../../docs/scroll-to.md) for parameters, visible bounds and errors. diff --git a/packages/dsh-plugin-browserskill/skill/SKILL.md b/packages/dsh-plugin-browserskill/skill/SKILL.md index 55a744ec..da74679e 100644 --- a/packages/dsh-plugin-browserskill/skill/SKILL.md +++ b/packages/dsh-plugin-browserskill/skill/SKILL.md @@ -51,15 +51,17 @@ Use `browser_inspect` action `observe` as the primary semantic page view. It ret text, and `@eN` refs. Prefer fresh refs over raw selectors. Refs invalidate after navigation and may also become stale after large DOM changes, so observe again before the next interaction. -Use `browser_interact` with `action=scroll-to` to bring an element into view. Its bounds are the -visible border-box portion in top-level viewport CSS pixels after ancestor clipping; partial -visibility is enough. Hidden or fully clipped targets fail. This does not test occlusion by other -elements. Use fresh element refs for iframe/shadow-root targets; selectors search the main document. - -Use `browser_interact` for click, hover, scroll-to, focus, blur, fill, select, and key actions. Focus and blur -explicitly enter or leave UI states triggered by focus changes. An observation marks a hover-only surface as `@e1 button "Products" [hover first: Shoes | Bags]`. The listed items are -labels, not usable refs: hover the trigger, observe again, then act on the revealed item's own ref. -Do not click the trigger itself unless the user wants the trigger's action. +Use `browser_interact` for click, hover, scroll-to, focus, blur, fill, select, press and wheel. +`scroll-to` reveals a target and returns its visible border-box bounds in top-level viewport CSS +pixels, clipped by ancestors. Partial visibility suffices; hidden/fully clipped targets fail. +This does not test occlusion. Use refs for iframe/shadow-root targets; selectors search the main document. + +`wheel` takes signed `deltaX`/`deltaY` (one nonzero). Optional `target` is scrolled into view first; +otherwise input lands at the viewport centre. Results echo input, not completed scrolling; observe afterwards. + +`focus`/`blur` enter or leave focus-triggered states. Hover-only surfaces appear as +`@e1 button "Products" [hover first: Shoes | Bags]`. Items are labels, not refs: +hover the trigger, observe, then use the revealed item's ref. Click the trigger only if its action is wanted. Escalate reading only as needed: diff --git a/packages/dsh-plugin-browserskill/src/browser-tools.ts b/packages/dsh-plugin-browserskill/src/browser-tools.ts index d002a9a1..4a4c484e 100644 --- a/packages/dsh-plugin-browserskill/src/browser-tools.ts +++ b/packages/dsh-plugin-browserskill/src/browser-tools.ts @@ -167,12 +167,13 @@ const BROWSER_TOOL_SPECS: BrowserToolSpec[] = [ { name: "browser_interact", description: - "Interact with an element in the active Agent Window tab. Actions: click, hover, scroll-to, focus, blur, fill, select, " + + "Interact with the active Agent Window tab. Actions: click, hover, wheel, scroll-to, focus, blur, fill, select, " + "press. click/hover/scroll-to/focus/blur/fill/select require target; fill also requires value; select requires " + - "values; press requires key and may optionally focus target first.", + "values; press requires key and may optionally focus target first. wheel requires a nonzero deltaX or deltaY and optionally accepts target; observe afterwards to check the response.", actions: { click: "interact.click", hover: "interact.hover", + wheel: "interact.wheel", "scroll-to": "interact.scroll-to", focus: "interact.focus", blur: "interact.blur", @@ -191,9 +192,14 @@ const BROWSER_TOOL_SPECS: BrowserToolSpec[] = [ modifiers: { type: "array", items: { type: "string", enum: ["alt", "ctrl", "meta", "shift"] }, - description: "Modifiers held during hover.", + description: "Modifiers held during hover or wheel input.", }, settleMs: { type: "integer", description: "Hover settle delay." }, + deltaX: { + type: "number", + description: "Horizontal wheel input in CSS pixels; defaults to 0.", + }, + deltaY: { type: "number", description: "Vertical wheel input in CSS pixels; defaults to 0." }, timeoutMs: TIMEOUT_MS_PARAM, values: { type: "array", diff --git a/packages/dsh-plugin-browserskill/src/phase-one-tools-interaction.ts b/packages/dsh-plugin-browserskill/src/phase-one-tools-interaction.ts index 50480215..2817367a 100644 --- a/packages/dsh-plugin-browserskill/src/phase-one-tools-interaction.ts +++ b/packages/dsh-plugin-browserskill/src/phase-one-tools-interaction.ts @@ -21,6 +21,106 @@ export function registerPhaseOneInteractionTools( ): void { const { registry } = deps; + register( + defineTool({ + name: "interact.wheel", + description: + "Send native wheel input at the viewport centre or an optional target. " + + "A target is scrolled into view first. Deltas are input CSS pixels, not measured scroll " + + "distance; observe afterwards to check the page's response.", + parameters: { + target: { + type: "string", + description: "Optional snapshot ref or main-document CSS selector.", + }, + session: SESSION_PARAM, + tabId: TAB_ID_PARAM, + deltaX: { type: "number", description: "Horizontal input in CSS pixels; defaults to 0." }, + deltaY: { type: "number", description: "Vertical input in CSS pixels; defaults to 0." }, + modifiers: { type: "array", items: { type: "string", enum: MODIFIERS } }, + timeoutMs: TIMEOUT_MS_PARAM, + }, + output: { + schema: { + type: "object", + additionalProperties: false, + properties: { + session: { type: "string", required: true }, + tabId: { type: "integer", required: true }, + x: { type: "number", required: true }, + y: { type: "number", required: true }, + deltaX: { type: "number", required: true }, + deltaY: { type: "number", required: true }, + }, + }, + render: (_args, value) => [ + { + type: "text", + text: `[session ${value.session}] wheel input (${value.deltaX}, ${value.deltaY}) sent at (${value.x}, ${value.y}) on tab ${value.tabId}`, + }, + ], + }, + async execute(args, exec) { + if (args.target !== undefined) requireNonEmpty(args.target, "target"); + requirePositive(args.timeoutMs, "timeoutMs"); + const deltaX = args.deltaX ?? 0; + const deltaY = args.deltaY ?? 0; + if (!Number.isFinite(deltaX) || !Number.isFinite(deltaY)) + throw new Error("wheel deltas must be finite numbers"); + if (deltaX === 0 && deltaY === 0) + throw new Error("at least one wheel delta must be non-zero"); + const sessionId = registry.resolve(args.session, "browser_interact(action=wheel)"); + const cmdArgs = [ + "wheel", + "--session", + sessionId, + "--delta-x", + String(deltaX), + "--delta-y", + String(deltaY), + ]; + appendTabId(cmdArgs, args.tabId); + if (args.modifiers?.length) cmdArgs.push("--modifiers", args.modifiers.join(",")); + if (args.timeoutMs !== undefined) cmdArgs.push("--timeout", `${args.timeoutMs}ms`); + if (args.target !== undefined) appendTarget(cmdArgs, args.target); + const reply = (await runtime.run( + exec, + cmdArgs, + "wheel", + sessionId, + runnerTimeout(deps, args.timeoutMs), + )) as { + tab_id: number; + x: number; + y: number; + delta_x: number; + delta_y: number; + }; + return { + session: sessionId, + tabId: reply.tab_id, + x: reply.x, + y: reply.y, + deltaX: reply.delta_x, + deltaY: reply.delta_y, + }; + }, + presentCall: (args) => ({ + card: "terminal", + title: runtime.commandLine([ + "wheel", + "--delta-x", + String(args.deltaX ?? 0), + "--delta-y", + String(args.deltaY ?? 0), + ...(args.target === undefined ? [] : [args.target]), + ]), + description: "Send native wheel input", + }), + presentResult: runtime.presentTerminalResult, + }), + ); + register( defineTool({ name: "interact.hover", diff --git a/packages/dsh-plugin-browserskill/tests/tools.test.ts b/packages/dsh-plugin-browserskill/tests/tools.test.ts index ab7686cb..b83a4bbc 100644 --- a/packages/dsh-plugin-browserskill/tests/tools.test.ts +++ b/packages/dsh-plugin-browserskill/tests/tools.test.ts @@ -102,6 +102,7 @@ const ACTION_ROUTES: Record = { "inspect.network": ["browser_inspect", "network"], "interact.click": ["browser_interact", "click"], "interact.hover": ["browser_interact", "hover"], + "interact.wheel": ["browser_interact", "wheel"], "interact.scroll-to": ["browser_interact", "scroll-to"], "interact.focus": ["browser_interact", "focus"], "interact.blur": ["browser_interact", "blur"], @@ -203,7 +204,17 @@ const EXPECTED_ACTIONS = { browser_session: ["start", "stop", "list"], browser_page: ["navigate", "back", "forward", "reload", "wait"], browser_inspect: ["observe", "snapshot", "html", "screenshot", "console", "network"], - browser_interact: ["click", "hover", "scroll-to", "focus", "blur", "fill", "select", "press"], + browser_interact: [ + "click", + "hover", + "wheel", + "scroll-to", + "focus", + "blur", + "fill", + "select", + "press", + ], browser_tabs: ["list", "create", "select", "close", "borrow", "return"], browser_assist: ["resize", "emulate", "request-help"], } as const; @@ -1408,3 +1419,62 @@ describe("observation action instrumentation timing", () => { observation.dispose(); }); }); + +describe("wheel action", () => { + it.each([ + undefined, + "@e3", + "#panel", + ])("maps target %s, signed deltas, modifiers and timeout", async (target) => { + const { tools, calls } = setup({ + "session start": START_REPLY("s1"), + wheel: { tab_id: 7, x: 10, y: 20, delta_x: -12.5, delta_y: 0 }, + }); + await startSession(tools); + expect( + await tools.get("interact.wheel")!.execute( + { + ...(target === undefined ? {} : { target }), + deltaX: -12.5, + modifiers: ["ctrl", "shift"], + tabId: 7, + timeoutMs: 150_000, + }, + makeExec(), + ), + ).toEqual({ session: "s1", tabId: 7, x: 10, y: 20, deltaX: -12.5, deltaY: 0 }); + expect(calls[1].args).toEqual([ + "wheel", + "--session", + "s1", + "--delta-x", + "-12.5", + "--delta-y", + "0", + "--tab-id", + "7", + "--modifiers", + "ctrl,shift", + "--timeout", + "150000ms", + ...(target === undefined ? [] : [target]), + ]); + expect(calls[1].options.timeoutMs).toBe(165_000); + }); + + it.each([ + {}, + { deltaX: 0, deltaY: 0 }, + { deltaY: NaN }, + { deltaX: Infinity }, + { target: " ", deltaY: 120 }, + { deltaY: 120, timeoutMs: 0 }, + { deltaY: 120, modifiers: ["invalid"] }, + { deltaY: 120, session: "foreign" }, + ])("rejects invalid or unowned arguments %j before invoking the CLI", async (args) => { + const { tools, calls } = setup({ "session start": START_REPLY("s1") }); + await startSession(tools); + await expect(tools.get("interact.wheel")!.execute(args, makeExec())).rejects.toThrow(); + expect(calls).toHaveLength(1); + }); +}); diff --git a/skill/SKILL.md b/skill/SKILL.md index 765b91cc..3eb44796 100644 --- a/skill/SKILL.md +++ b/skill/SKILL.md @@ -55,7 +55,7 @@ Use this default loop: ```text bsk navigate --session bsk observe --session -bsk click|hover|scroll-to|focus|blur|fill|select|press ... --session +bsk click|hover|wheel|scroll-to|focus|blur|fill|select|press ... --session bsk observe --session # after navigation or a meaningful DOM change ``` @@ -66,6 +66,11 @@ ancestor clipping. Partial visibility is enough; hidden or fully clipped targets `permission_denied` and `data.reason=element_not_visible`. This does not test occlusion by other elements. For a specific tab or deadline: `bsk scroll-to @e3 --session --tab-id 42 --timeout 5s`. +`bsk wheel --delta-y -120 --session ` sends native wheel input at the viewport centre. +Add an optional ref/selector to target an element (scrolled into view first). Both delta axes +accept signed numbers and default to zero; at least one must be nonzero. The result echoes +input, not actual scroll distance or completion. Observe afterwards to check the page's response. + `bsk focus ` explicitly focuses a target; `bsk blur ` removes focus and reports whether it was focused. Use these for UI states triggered by focus changes. @@ -124,7 +129,7 @@ This list of names is complete. Never invent a command outside it; read session start|stop|list browsers status doctor update logs navigate navigate-back navigate-forward reload wait-for-navigation wait-ms observe snapshot get-html screenshot console network -click hover scroll-to focus blur fill select press evaluate +click hover wheel scroll-to focus blur fill select press evaluate tab list|create|close|select|borrow|return window resize emulate upload download request-help record start|stop ```