feat(mcp): take mouse and keyboard in the task view when Sai isn't working

mcp/src/__tests__/app-view.test.ts

@@ -149,6 +149,23 @@ describe('MCP Apps task view (server)', () => {
     expect(gone.structuredContent).toMatchObject({ expired: true })
   })
 
+  it('archives a task once a newer one starts on its machine, even in another session', async () => {
+    const wait = vi.fn().mockResolvedValue({ status: 'idle', events: [], text: '', cursor: 'c' })
+    const tasks = new TaskLog()
+    const { client } = await connect(wait, undefined, tasks)
+    const first = tasks.start('s1', 'm1')
+    tasks.start('s9', 'm2') // another machine: no effect
+    expect(tasks.state(first, 's1')).toBe('live')
+
+    tasks.start('s2', 'm1') // new_session on the same machine
+    const r = await client.callTool({
+      name: 'sai_view_poll',
+      arguments: { session_id: 's1', task_id: first },
+    })
+    expect(wait).not.toHaveBeenCalled()
+    expect(r.structuredContent).toMatchObject({ archived: true })
+  })
+
   it('accepts only a JPEG data URL as the saved frame', async () => {
     const tasks = new TaskLog()
     const { client } = await connect(vi.fn(), undefined, tasks)

mcp/src/server.ts

@@ -156,7 +156,7 @@ export function createServer(): McpServer {
       run(async () => {
         const started = await tools.start(args)
         // Which task this call's view follows (the session outlives it).
-        const taskId = taskLog.start(started.session_id)
+        const taskId = taskLog.start(started.session_id, started.machine_id)
         // For hosts that can't render the view inline: the same page in a browser.
         const watchUrl = await localView
           ?.urlFor(started.session_id, started.machine_id, taskId)

mcp/src/task-log.ts

@@ -5,6 +5,10 @@
  * every task its views show, so it knows their order: a view of an older task
  * gets the snapshot that view saved when its task ended, not the live turn.
  *
+ * A newer task on the same machine also archives a view, even in another
+ * session (`new_session`): the person using the computer from an ended task's
+ * view must stop once Sai works on it again.
+ *
  * In memory only: after a restart an old view is told its task expired.
  */
 import { randomBytes } from 'node:crypto'
@@ -21,6 +25,7 @@ export interface TaskSnapshot {
 
 interface TaskRecord {
   sessionId: string
+  machineId?: string
   saved?: TaskSnapshot
 }
 
@@ -29,24 +34,31 @@ const MAX_TASKS = 50
 export class TaskLog {
   private readonly tasks = new Map<string, TaskRecord>()
   private readonly latest = new Map<string, string>()
+  private readonly latestOnMachine = new Map<string, string>()
 
-  /** A new task on `sessionId`: it becomes that session's live one. */
-  start(sessionId: string): string {
+  /** A new task on `sessionId`: it becomes that session's (and machine's) live one. */
+  start(sessionId: string, machineId?: string): string {
     const taskId = randomBytes(9).toString('base64url')
-    this.tasks.set(taskId, { sessionId })
+    this.tasks.set(taskId, { sessionId, ...(machineId && { machineId }) })
     this.latest.set(sessionId, taskId)
+    if (machineId) this.latestOnMachine.set(machineId, taskId)
     while (this.tasks.size > MAX_TASKS) {
       const oldest = this.tasks.keys().next().value!
       this.tasks.delete(oldest)
     }
     return taskId
   }
 
-  /** `live`: still its session's newest task; `archived`: a newer one started; `expired`: unknown. */
+  /**
+   * `live`: still its session's and machine's newest task; `archived`: a newer
+   * one started on either; `expired`: unknown.
+   */
   state(taskId: string, sessionId: string): 'live' | 'archived' | 'expired' {
     const task = this.tasks.get(taskId)
     if (!task || task.sessionId !== sessionId) return 'expired'
-    return this.latest.get(sessionId) === taskId ? 'live' : 'archived'
+    if (this.latest.get(sessionId) !== taskId) return 'archived'
+    if (task.machineId && this.latestOnMachine.get(task.machineId) !== taskId) return 'archived'
+    return 'live'
   }
 
   saved(taskId: string): TaskSnapshot | undefined {

mcp/src/ui/__tests__/gating.test.ts

@@ -0,0 +1,52 @@
+import { describe, expect, it } from 'vitest'
+
+import {
+  ANOTHER_TASK_NOTE,
+  UNKNOWN_TASK_NOTE,
+  initialView,
+  inputAllowed,
+  useEnd,
+  useStopsOnLoss,
+  type TaskView,
+} from '../view-model.js'
+
+const at = (status: TaskView['status']): TaskView => ({ ...initialView, sessionId: 's1', status })
+
+describe('task view input gating', () => {
+  it('takes input while the task waits on the person, not while Sai works', () => {
+    expect(inputAllowed(at('needs_approval'), false)).toBe(true)
+    // decide() sets running before it calls sai_task_approve.
+    expect(inputAllowed(at('running'), false)).toBe(false)
+    expect(inputAllowed(at('starting'), false)).toBe(false)
+  })
+
+  it('after the task, takes input only while the person uses the computer', () => {
+    for (const status of ['idle', 'error'] as const) {
+      expect(inputAllowed(at(status), true)).toBe(true)
+      // Done: input goes off at once, not when the screen closes.
+      expect(inputAllowed(at(status), false)).toBe(false)
+    }
+  })
+
+  it('stops using the computer once Sai starts another task on it', () => {
+    expect(useEnd({ archived: true })).toBe(ANOTHER_TASK_NOTE)
+    // A new turn on the same session (e.g. from another client).
+    expect(useEnd({ status: 'running' })).toBe(ANOTHER_TASK_NOTE)
+    expect(useEnd({ status: 'needs_approval' })).toBe(ANOTHER_TASK_NOTE)
+    expect(useEnd({ expired: true })).toBe(UNKNOWN_TASK_NOTE)
+  })
+
+  it('keeps using the computer while the session stays done', () => {
+    expect(useEnd({ status: 'idle' })).toBeUndefined()
+    expect(useEnd({ status: 'error' })).toBeUndefined()
+  })
+
+  it('stops using it when the computer has no tunnel or is unreachable, after the task only', () => {
+    expect(useStopsOnLoss(at('idle'), 'no_tunnel')).toBe(true)
+    expect(useStopsOnLoss(at('idle'), 'unreachable')).toBe(true)
+    expect(useStopsOnLoss(at('idle'), 'other')).toBe(false)
+    // While the task runs, the view waits for the computer instead.
+    expect(useStopsOnLoss(at('running'), 'no_tunnel')).toBe(false)
+    expect(useStopsOnLoss(at('needs_approval'), 'unreachable')).toBe(false)
+  })
+})

mcp/src/ui/__tests__/input.test.ts

@@ -0,0 +1,143 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+
+import { bindScreenInput } from '../input.js'
+
+type State = Guacamole.Mouse.State
+type Pointer = {
+  onmousedown: ((s: State) => void) | null
+  onmousemove: ((s: State) => void) | null
+  onmouseup: ((s: State) => void) | null
+}
+
+/** Guacamole's input classes, reduced to the handlers the binding sets. */
+function fakeGuacamole() {
+  const made: { keyboard?: FakeKeyboard; pointers: Pointer[] } = { pointers: [] }
+  class FakeKeyboard {
+    onkeydown: ((k: number) => boolean | void) | null = null
+    onkeyup: ((k: number) => void) | null = null
+    down = new Set<number>()
+    constructor() {
+      made.keyboard = this
+    }
+    press(k: number): boolean | void {
+      this.down.add(k)
+      return this.onkeydown?.(k)
+    }
+    reset(): void {
+      for (const k of this.down) this.onkeyup?.(k)
+      this.down.clear()
+    }
+  }
+  class FakePointer implements Pointer {
+    onmousedown: Pointer['onmousedown'] = null
+    onmousemove: Pointer['onmousemove'] = null
+    onmouseup: Pointer['onmouseup'] = null
+    constructor() {
+      made.pointers.push(this)
+    }
+  }
+  const Mouse = Object.assign(class extends FakePointer {}, {
+    Touchscreen: class extends FakePointer {},
+  })
+  const G = { Keyboard: FakeKeyboard, Mouse } as unknown as typeof Guacamole
+  return { G, made }
+}
+
+function fakeElement() {
+  const el = new EventTarget() as EventTarget & Record<string, unknown>
+  Object.assign(el, {
+    tabIndex: -1,
+    style: {} as Record<string, string>,
+    attrs: new Map<string, string>(),
+    focus: vi.fn(),
+    blur: vi.fn(),
+    setAttribute(k: string, v: string) {
+      ;(el.attrs as Map<string, string>).set(k, v)
+    },
+    removeAttribute(k: string) {
+      ;(el.attrs as Map<string, string>).delete(k)
+    },
+  })
+  return el as unknown as HTMLElement & { attrs: Map<string, string> }
+}
+
+const newClient = () => ({
+  sendKeyEvent: vi.fn<(pressed: number, keysym: number) => void>(),
+  sendMouseState: vi.fn<(state: State, scale?: boolean) => void>(),
+})
+let client: ReturnType<typeof newClient>
+beforeEach(() => {
+  vi.stubGlobal('window', new EventTarget())
+  client = newClient()
+})
+afterEach(() => vi.unstubAllGlobals())
+
+describe('bindScreenInput', () => {
+  it('sends nothing until input is turned on, and leaves the keys to the page', () => {
+    const { G, made } = fakeGuacamole()
+    const el = fakeElement()
+    bindScreenInput(G, el, client)
+    expect(made.keyboard!.press(65)).toBe(true)
+    made.pointers[0]!.onmousedown!({ x: 1, y: 2, left: true })
+    expect(client.sendKeyEvent).not.toHaveBeenCalled()
+    expect(client.sendMouseState).not.toHaveBeenCalled()
+    expect(el.tabIndex).toBe(-1)
+  })
+
+  it('forwards mouse, touch and keys once on, scaled to the computer', () => {
+    const { G, made } = fakeGuacamole()
+    const el = fakeElement()
+    bindScreenInput(G, el, client).set(true)
+    expect(el.tabIndex).toBe(0)
+    expect(el.attrs.get('role')).toBe('application')
+    made.pointers[0]!.onmousedown!({ x: 10, y: 20, left: true })
+    made.pointers[1]!.onmousemove!({ x: 11, y: 21 })
+    expect(client.sendMouseState).toHaveBeenNthCalledWith(1, { x: 10, y: 20, left: true }, true)
+    expect(client.sendMouseState).toHaveBeenNthCalledWith(2, { x: 11, y: 21 }, true)
+    expect(made.keyboard!.press(65)).toBe(false)
+    expect(client.sendKeyEvent).toHaveBeenCalledWith(1, 65)
+  })
+
+  it('lets go of held keys and buttons when turned off, then stops sending', () => {
+    const { G, made } = fakeGuacamole()
+    const el = fakeElement()
+    const input = bindScreenInput(G, el, client)
+    input.set(true)
+    made.keyboard!.press(0xffe3)
+    made.pointers[0]!.onmousedown!({ x: 5, y: 6, left: true })
+    client.sendKeyEvent.mockClear()
+    client.sendMouseState.mockClear()
+    input.set(false)
+    expect(client.sendKeyEvent).toHaveBeenCalledWith(0, 0xffe3)
+    expect(client.sendMouseState).toHaveBeenCalledWith({ x: 5, y: 6 }, true)
+    expect(el.tabIndex).toBe(-1)
+    expect(el.attrs.has('role')).toBe(false)
+    client.sendKeyEvent.mockClear()
+    made.keyboard!.press(66)
+    expect(client.sendKeyEvent).not.toHaveBeenCalled()
+  })
+
+  it('releases held keys when the page loses focus', () => {
+    const { G, made } = fakeGuacamole()
+    bindScreenInput(G, fakeElement(), client).set(true)
+    made.keyboard!.press(0xffe9)
+    window.dispatchEvent(new Event('blur'))
+    expect(client.sendKeyEvent).toHaveBeenLastCalledWith(0, 0xffe9)
+  })
+
+  it('gives the keyboard back on Control+Alt+Escape', () => {
+    const { G, made } = fakeGuacamole()
+    const el = fakeElement()
+    bindScreenInput(G, el, client).set(true)
+    made.keyboard!.press(0xffe3)
+    const event = Object.assign(new Event('keydown', { cancelable: true }), {
+      key: 'Escape',
+      ctrlKey: true,
+      altKey: true,
+    })
+    el.dispatchEvent(event)
+    expect(event.defaultPrevented).toBe(true)
+    expect(el.blur).toHaveBeenCalled()
+    expect(client.sendKeyEvent).toHaveBeenLastCalledWith(0, 0xffe3)
+  })
+})

mcp/src/ui/input.ts

@@ -0,0 +1,110 @@
+/**
+ * Mouse, touch and keyboard from the live screen to the computer, bound the
+ * way the portal Playground binds them (sai-platform
+ * `src/playground/guacamole/input.ts`). The keyboard listens on the screen
+ * element only, never the document, so the host's chat box keeps its keys.
+ *
+ * Starts off. `set(false)` first lets go of any key or button still held, so
+ * the computer is never left with a key down.
+ */
+
+export interface ScreenInput {
+  set: (on: boolean) => void
+  dispose: () => void
+}
+
+type MouseLike = {
+  onmousedown: ((s: Guacamole.Mouse.State) => void) | null
+  onmousemove: ((s: Guacamole.Mouse.State) => void) | null
+  onmouseup: ((s: Guacamole.Mouse.State) => void) | null
+}
+
+export function bindScreenInput(
+  G: typeof Guacamole,
+  element: HTMLElement,
+  client: Pick<Guacamole.Client, 'sendKeyEvent' | 'sendMouseState'>,
+): ScreenInput {
+  const keyboard = new G.Keyboard(element)
+  const pointers: MouseLike[] = [new G.Mouse(element), new G.Mouse.Touchscreen(element)]
+  let on = false
+  let last: Guacamole.Mouse.State = { x: 0, y: 0 }
+
+  const send = (state: Guacamole.Mouse.State): void => {
+    if (!on) return
+    last = { ...state }
+    // `true`: Guacamole maps the scaled screen box back to the computer's pixels.
+    client.sendMouseState(state, true)
+  }
+  for (const p of pointers) p.onmousedown = p.onmousemove = p.onmouseup = send
+  keyboard.onkeydown = (key: number) => {
+    if (!on) return true
+    client.sendKeyEvent(1, key)
+    return false
+  }
+  keyboard.onkeyup = (key: number) => {
+    if (on) client.sendKeyEvent(0, key)
+  }
+
+  const release = (): void => {
+    if (!on) return
+    keyboard.reset()
+    if (last.left || last.middle || last.right)
+      client.sendMouseState({ x: last.x, y: last.y }, true)
+    last = { x: last.x, y: last.y }
+  }
+  const focus = (): void => {
+    if (on) element.focus({ preventScroll: true })
+  }
+  // Every key goes to the computer while the screen has focus, Tab included:
+  // Ctrl+Alt+Esc hands the keyboard back to the page.
+  const exit = (event: KeyboardEvent): void => {
+    if (event.key !== 'Escape' || !event.ctrlKey || !event.altKey) return
+    event.preventDefault()
+    event.stopImmediatePropagation()
+    release()
+    element.blur()
+  }
+  const noMenu = (event: Event): void => {
+    if (on) event.preventDefault()
+  }
+  element.addEventListener('keydown', exit, true)
+  element.addEventListener('pointerdown', focus)
+  element.addEventListener('blur', release)
+  element.addEventListener('contextmenu', noMenu)
+  window.addEventListener('blur', release)
+
+  const set = (next: boolean): void => {
+    if (next === on) return
+    if (!next) release()
+    on = next
+    element.tabIndex = on ? 0 : -1
+    element.style.cursor = on ? '' : 'default'
+    element.style.touchAction = on ? 'none' : ''
+    if (on) {
+      element.setAttribute('role', 'application')
+      element.setAttribute(
+        'aria-label',
+        'Remote computer. Click to type. Press Control Alt Escape to return to the page.',
+      )
+    } else {
+      element.removeAttribute('role')
+      element.removeAttribute('aria-label')
+      element.blur()
+    }
+  }
+  set(false)
+
+  return {
+    set,
+    dispose: () => {
+      set(false)
+      keyboard.onkeydown = keyboard.onkeyup = null
+      for (const p of pointers) p.onmousedown = p.onmousemove = p.onmouseup = null
+      element.removeEventListener('keydown', exit, true)
+      element.removeEventListener('pointerdown', focus)
+      element.removeEventListener('blur', release)
+      element.removeEventListener('contextmenu', noMenu)
+      window.removeEventListener('blur', release)
+    },
+  }
+}