fix(browser): composite background agent tabs, actionable refusals, timed pointer paths (#8291)

* fix(browser): keep the agent tab composited when no panel shows it

A tab view that is in no window has no compositor surface, so CDP mouse input on it never acknowledges and screenshots never capture. The agent's tab was left that way whenever the user viewed another tab or had not opened the browser panel: clicks failed or stalled five seconds and screenshots failed. Each chat's agent tab is now parked invisibly in the main window, adopted in place when the panel shows it, and released on every tab teardown path.

* improvement(browser): name the controls a refused click can use next

A click refused because an overlay covers the target now lists the overlay's own snapshot controls, so the agent dismisses it and retries the same id without another snapshot. A click that lands on a nested control names that control's id. Only real overlays (dialogs, modals, fixed or sticky layers) list controls.

* feat(browser): route drags and coordinate hovers along timed pointer paths

browser_drag takes optional via points and a durationMs, and browser_hover can target a viewport point with the same route, moving with no button pressed. Pages that react to pointer speed or position (sliders, physics, canvas hover, cursor-following games) can now be driven the way Claude computer use and OpenAI's CUA allow. A plain drag keeps its exact previous pace; batches refuse timed paths so they cannot outlast the batch watchdog.

* fix(browser): honor cancellation on pointer routes and keep agent tabs tracked across scopes

- Timed drags and hover routes stop as soon as their tool call is cancelled;
  a drag still releases the button through its existing cleanup.
- A coordinate hover refuses durationMs without via points instead of
  silently moving straight to the target.
- A scope migration carries the chat's agent tab registration with it, so the
  migrated chat's old agent tab is released when it switches tabs.
- Overlay controls are found through shadow roots and with any tag casing.

* fix(browser): cancel drags during their settle and keep parked agent tabs in the main window

- An already-aborted drag sends no input, and an abort during the settle hold
  takes the cancellation cleanup instead of dropping.
- A coordinate hover re-checks the active tab right before it moves.
- Parking an agent tab hands keyboard focus back to whichever of the Sim
  renderer or the visible page held it.
- An agent tab detached from a secondary window is parked in the main window,
  which outlives it.
- A framed target covered by a parent-page overlay lists that overlay's controls.

* fix(browser): refuse a coordinate hover across a navigation

The coordinate hover captures the navigation epoch before its page probes and rejects a cross-document navigation before moving the pointer, as browser_click_at and browser_drag do. The overlay helper skips the empty registry slots earlier snapshots leave.

Author: waleedlatif1

apps/desktop/src/main/browser-agent/cdp.test.ts

@@ -15,10 +15,13 @@ import {
   captureScreenshot,
   clickAt,
   consumeAgentContextMenu,
+  dragPointer,
   ensureInstrumented,
   evaluateInIsolatedFrame,
   insertText,
+  movePointer,
   PRIMARY_CLICK,
+  pointerPathSteps,
   releaseFileInput,
   resolveFileInput,
   setColorScheme,
@@ -258,6 +261,122 @@ describe('browser-agent CDP instrumentation', () => {
     ])
   })
 
+  it('stops a pointer route as soon as it is aborted', async () => {
+    const contents = new WebContentsView().webContents
+    const moves = () =>
+      vi
+        .mocked(contents.debugger.sendCommand)
+        .mock.calls.filter(([method]) => String(method).startsWith('Input.dispatchMouseEvent'))
+    const aborted = new AbortController()
+    aborted.abort()
+    await expect(
+      movePointer(contents, { via: [], durationMs: null }, { x: 5, y: 5 }, aborted.signal)
+    ).rejects.toMatchObject({ name: 'AbortError' })
+    expect(moves()).toHaveLength(0)
+
+    vi.useFakeTimers()
+    try {
+      const controller = new AbortController()
+      const route = movePointer(
+        contents,
+        { via: [{ x: 0, y: 0 }], durationMs: 5_000 },
+        { x: 500, y: 0 },
+        controller.signal
+      )
+      const settled = expect(route).rejects.toMatchObject({ name: 'AbortError' })
+      await vi.advanceTimersByTimeAsync(100)
+      const sentBeforeAbort = moves().length
+      controller.abort()
+      await settled
+      expect(moves()).toHaveLength(sentBeforeAbort)
+    } finally {
+      vi.useRealTimers()
+    }
+  })
+
+  it('sends nothing for an already-aborted drag and cancels one aborted while it settles', async () => {
+    const contents = new WebContentsView().webContents
+    const mouse = () =>
+      vi
+        .mocked(contents.debugger.sendCommand)
+        .mock.calls.filter(([method]) => method === 'Input.dispatchMouseEvent')
+        .map(([, params]) => toRecord(params).type)
+    const aborted = new AbortController()
+    aborted.abort()
+    await expect(
+      dragPointer(contents, { x: 0, y: 0 }, { x: 50, y: 0 }, undefined, aborted.signal)
+    ).rejects.toMatchObject({ name: 'AbortError' })
+    expect(contents.debugger.sendCommand).not.toHaveBeenCalled()
+
+    vi.useFakeTimers()
+    try {
+      const controller = new AbortController()
+      const drag = dragPointer(
+        contents,
+        { x: 0, y: 0 },
+        { x: 50, y: 0 },
+        undefined,
+        controller.signal
+      )
+      const settled = expect(drag).rejects.toMatchObject({ name: 'AbortError' })
+      // The default route takes 13 moves 20 ms apart, then a 120 ms settle hold.
+      await vi.advanceTimersByTimeAsync(300)
+      controller.abort()
+      await settled
+      expect(mouse().at(-1)).toBe('mouseReleased')
+    } finally {
+      vi.useRealTimers()
+    }
+  })
+
+  it('releases the button when a timed drag is aborted mid-route', async () => {
+    const contents = new WebContentsView().webContents
+    const types = () =>
+      vi
+        .mocked(contents.debugger.sendCommand)
+        .mock.calls.filter(([method]) => method === 'Input.dispatchMouseEvent')
+        .map(([, params]) => toRecord(params).type)
+    vi.useFakeTimers()
+    try {
+      const controller = new AbortController()
+      const drag = dragPointer(
+        contents,
+        { x: 0, y: 0 },
+        { x: 500, y: 0 },
+        { via: [], durationMs: 5_000 },
+        controller.signal
+      )
+      const settled = expect(drag).rejects.toMatchObject({ name: 'AbortError' })
+      await vi.advanceTimersByTimeAsync(100)
+      controller.abort()
+      await settled
+      expect(types().at(-1)).toBe('mouseReleased')
+      expect(types().filter((type) => type === 'mouseMoved').length).toBeLessThan(20)
+    } finally {
+      vi.useRealTimers()
+    }
+  })
+
+  it('keeps the default drag pace and lands exactly on every via point', () => {
+    const direct = pointerPathSteps({ x: 0, y: 0 }, { via: [], durationMs: null }, { x: 120, y: 0 })
+    expect(direct.stepDelayMs).toBe(20)
+    expect(direct.points).toHaveLength(12)
+    expect(direct.points[0]).toEqual({ x: 10, y: 0 })
+    expect(direct.points[11]).toEqual({ x: 120, y: 0 })
+
+    const routed = pointerPathSteps(
+      { x: 0, y: 0 },
+      { via: [{ x: 100, y: 0 }], durationMs: 800 },
+      { x: 100, y: 300 }
+    )
+    expect(routed.points).toContainEqual({ x: 100, y: 0 })
+    expect(routed.points[routed.points.length - 1]).toEqual({ x: 100, y: 300 })
+    expect(routed.points.length * routed.stepDelayMs).toBeCloseTo(800)
+    // The longer second segment gets about three times the steps of the first.
+    const corner = routed.points.findIndex((point) => point.x === 100 && point.y === 0)
+    expect(routed.points.length - 1 - corner).toBeGreaterThan(corner * 2)
+  })
+
   it('holds the button down for holdMs before releasing it', async () => {
     const contents = new WebContentsView().webContents
     const types = () =>

apps/desktop/src/main/browser-agent/cdp.ts

@@ -11,7 +11,7 @@
 import type { BrowserTheme } from '@sim/browser-protocol'
 import { createLogger } from '@sim/logger'
 import { getErrorMessage } from '@sim/utils/errors'
-import { interruptibleSleep, sleep } from '@sim/utils/helpers'
+import { interruptibleSleep } from '@sim/utils/helpers'
 import { isRecordLike } from '@sim/utils/object'
 import type { NativeImage, WebContents, WebFrameMain } from 'electron'
 
@@ -948,6 +948,91 @@ export async function moveMouse(contents: WebContents, x: number, y: number): Pr
   })
 }
 
+/** A point in CSS viewport pixels. */
+export interface ViewportPoint {
+  x: number
+  y: number
+}
+
+/** The points a pointer passes through on its way, and how long the whole movement takes. */
+export interface PointerPath {
+  via: ViewportPoint[]
+  /** Total movement time; null keeps the default brisk pace. */
+  durationMs: number | null
+}
+
+export const DIRECT_PATH: PointerPath = { via: [], durationMs: null }
+
+const DEFAULT_PATH_STEPS = 12
+const DEFAULT_PATH_STEP_MS = 20
+/** One display frame, so a timed movement looks continuous to animation-driven pages. */
+const TIMED_PATH_STEP_MS = 16
+
+/**
+ * The moves from `from` through `path.via` to `to`, and the pause after each. A direct path
+ * keeps the default 12 moves 20 ms apart. A path with via points or a duration moves one frame
+ * at a time, shares the steps across segments by length, and lands exactly on every via point.
+ */
+export function pointerPathSteps(
+  from: ViewportPoint,
+  path: PointerPath,
+  to: ViewportPoint
+): { points: ViewportPoint[]; stepDelayMs: number } {
+  const lerp = (a: ViewportPoint, b: ViewportPoint, t: number): ViewportPoint => ({
+    x: a.x + (b.x - a.x) * t,
+    y: a.y + (b.y - a.y) * t,
+  })
+  if (path.via.length === 0 && path.durationMs === null) {
+    const points: ViewportPoint[] = []
+    for (let step = 1; step <= DEFAULT_PATH_STEPS; step++) {
+      points.push(lerp(from, to, step / DEFAULT_PATH_STEPS))
+    }
+    return { points, stepDelayMs: DEFAULT_PATH_STEP_MS }
+  }
+  const vertices = [from, ...path.via, to]
+  const durationMs = path.durationMs ?? DEFAULT_PATH_STEPS * DEFAULT_PATH_STEP_MS
+  const lengths = vertices
+    .slice(1)
+    .map((vertex, index) => Math.hypot(vertex.x - vertices[index].x, vertex.y - vertices[index].y))
+  const totalLength = lengths.reduce((sum, length) => sum + length, 0)
+  const totalSteps = Math.max(lengths.length, Math.round(durationMs / TIMED_PATH_STEP_MS))
+  const points: ViewportPoint[] = []
+  lengths.forEach((length, index) => {
+    const share = totalLength > 0 ? length / totalLength : 1 / lengths.length
+    const segmentSteps = Math.max(1, Math.round(totalSteps * share))
+    for (let step = 1; step <= segmentSteps; step++) {
+      points.push(lerp(vertices[index], vertices[index + 1], step / segmentSteps))
+    }
+  })
+  return { points, stepDelayMs: durationMs / points.length }
+}
+
+/**
+ * Moves the pointer with no button pressed through `path.via` to `to`, starting from the first
+ * via point (or `to` itself for a direct move), for hover effects that follow the cursor.
+ */
+export async function movePointer(
+  contents: WebContents,
+  path: PointerPath,
+  to: ViewportPoint,
+  signal?: AbortSignal
+): Promise<void> {
+  const [start, ...rest] = [...path.via, to]
+  signal?.throwIfAborted()
+  await moveMouse(contents, start.x, start.y)
+  if (rest.length === 0) return
+  const { points, stepDelayMs } = pointerPathSteps(
+    start,
+    { via: rest.slice(0, -1), durationMs: path.durationMs },
+    to
+  )
+  for (const point of points) {
+    await interruptibleSleep(stepDelayMs, signal)
+    signal?.throwIfAborted()
+    await moveMouse(contents, point.x, point.y)
+  }
+}
+
 /** One trusted click gesture: which button, how many presses, and held modifiers. */
 export interface PointerClick {
   button: 'left' | 'right' | 'middle'
@@ -1075,11 +1160,13 @@ export async function clickAt(
  */
 export async function dragPointer(
   contents: WebContents,
-  from: { x: number; y: number },
-  to: { x: number; y: number },
-  steps = 12,
-  stepDelayMs = 20
+  from: ViewportPoint,
+  to: ViewportPoint,
+  path: PointerPath = DIRECT_PATH,
+  signal?: AbortSignal
 ): Promise<{ nativeDragIntercepted: boolean }> {
+  signal?.throwIfAborted()
+  const { points, stepDelayMs } = pointerPathSteps(from, path, to)
   const interception: DragInterception = { intercepted: false, data: null }
   dragInterceptionsByContents.set(contents, interception)
   let interceptEnabled = false
@@ -1124,17 +1211,19 @@ export async function dragPointer(
     })
     // Small first nudge so libraries with a start threshold (commonly 3-8px)
     // register the drag before the pointer sweeps across the page.
-    await dragMove(from.x + Math.sign(to.x - from.x || 1) * 4, from.y + 2)
-    await sleep(stepDelayMs)
-    const stepCount = Math.max(2, steps)
-    for (let step = 1; step <= stepCount; step++) {
-      const progress = step / stepCount
-      await dragMove(from.x + (to.x - from.x) * progress, from.y + (to.y - from.y) * progress)
-      await sleep(stepDelayMs)
+    const heading = points[0] ?? to
+    await dragMove(from.x + Math.sign(heading.x - from.x || 1) * 4, from.y + 2)
+    await interruptibleSleep(stepDelayMs, signal)
+    for (const point of points) {
+      signal?.throwIfAborted()
+      await dragMove(point.x, point.y)
+      await interruptibleSleep(stepDelayMs, signal)
     }
+    signal?.throwIfAborted()
     // Hold over the target so drop zones running enter/over animations settle
     // before the release lands.
-    await sleep(120)
+    await interruptibleSleep(120, signal)
+    signal?.throwIfAborted()
     if (interception.intercepted && interception.data) {
       await sendInput(contents, 'Input.dispatchDragEvent', {
         type: 'drop',