feat(mcp): take mouse and keyboard in the task view when Sai isn't working

mcp/src/ui/__tests__/input.test.ts

@@ -0,0 +1,143 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+
+import { bindScreenInput } from '../input.js'
+
+type State = Guacamole.Mouse.State
+type Pointer = {
+  onmousedown: ((s: State) => void) | null
+  onmousemove: ((s: State) => void) | null
+  onmouseup: ((s: State) => void) | null
+}
+
+/** Guacamole's input classes, reduced to the handlers the binding sets. */
+function fakeGuacamole() {
+  const made: { keyboard?: FakeKeyboard; pointers: Pointer[] } = { pointers: [] }
+  class FakeKeyboard {
+    onkeydown: ((k: number) => boolean | void) | null = null
+    onkeyup: ((k: number) => void) | null = null
+    down = new Set<number>()
+    constructor() {
+      made.keyboard = this
+    }
+    press(k: number): boolean | void {
+      this.down.add(k)
+      return this.onkeydown?.(k)
+    }
+    reset(): void {
+      for (const k of this.down) this.onkeyup?.(k)
+      this.down.clear()
+    }
+  }
+  class FakePointer implements Pointer {
+    onmousedown: Pointer['onmousedown'] = null
+    onmousemove: Pointer['onmousemove'] = null
+    onmouseup: Pointer['onmouseup'] = null
+    constructor() {
+      made.pointers.push(this)
+    }
+  }
+  const Mouse = Object.assign(class extends FakePointer {}, {
+    Touchscreen: class extends FakePointer {},
+  })
+  const G = { Keyboard: FakeKeyboard, Mouse } as unknown as typeof Guacamole
+  return { G, made }
+}
+
+function fakeElement() {
+  const el = new EventTarget() as EventTarget & Record<string, unknown>
+  Object.assign(el, {
+    tabIndex: -1,
+    style: {} as Record<string, string>,
+    attrs: new Map<string, string>(),
+    focus: vi.fn(),
+    blur: vi.fn(),
+    setAttribute(k: string, v: string) {
+      ;(el.attrs as Map<string, string>).set(k, v)
+    },
+    removeAttribute(k: string) {
+      ;(el.attrs as Map<string, string>).delete(k)
+    },
+  })
+  return el as unknown as HTMLElement & { attrs: Map<string, string> }
+}
+
+const newClient = () => ({
+  sendKeyEvent: vi.fn<(pressed: number, keysym: number) => void>(),
+  sendMouseState: vi.fn<(state: State, scale?: boolean) => void>(),
+})
+let client: ReturnType<typeof newClient>
+beforeEach(() => {
+  vi.stubGlobal('window', new EventTarget())
+  client = newClient()
+})
+afterEach(() => vi.unstubAllGlobals())
+
+describe('bindScreenInput', () => {
+  it('sends nothing until input is turned on, and leaves the keys to the page', () => {
+    const { G, made } = fakeGuacamole()
+    const el = fakeElement()
+    bindScreenInput(G, el, client)
+    expect(made.keyboard!.press(65)).toBe(true)
+    made.pointers[0]!.onmousedown!({ x: 1, y: 2, left: true })
+    expect(client.sendKeyEvent).not.toHaveBeenCalled()
+    expect(client.sendMouseState).not.toHaveBeenCalled()
+    expect(el.tabIndex).toBe(-1)
+  })
+
+  it('forwards mouse, touch and keys once on, scaled to the computer', () => {
+    const { G, made } = fakeGuacamole()
+    const el = fakeElement()
+    bindScreenInput(G, el, client).set(true)
+    expect(el.tabIndex).toBe(0)
+    expect(el.attrs.get('role')).toBe('application')
+    made.pointers[0]!.onmousedown!({ x: 10, y: 20, left: true })
+    made.pointers[1]!.onmousemove!({ x: 11, y: 21 })
+    expect(client.sendMouseState).toHaveBeenNthCalledWith(1, { x: 10, y: 20, left: true }, true)
+    expect(client.sendMouseState).toHaveBeenNthCalledWith(2, { x: 11, y: 21 }, true)
+    expect(made.keyboard!.press(65)).toBe(false)
+    expect(client.sendKeyEvent).toHaveBeenCalledWith(1, 65)
+  })
+
+  it('lets go of held keys and buttons when turned off, then stops sending', () => {
+    const { G, made } = fakeGuacamole()
+    const el = fakeElement()
+    const input = bindScreenInput(G, el, client)
+    input.set(true)
+    made.keyboard!.press(0xffe3)
+    made.pointers[0]!.onmousedown!({ x: 5, y: 6, left: true })
+    client.sendKeyEvent.mockClear()
+    client.sendMouseState.mockClear()
+    input.set(false)
+    expect(client.sendKeyEvent).toHaveBeenCalledWith(0, 0xffe3)
+    expect(client.sendMouseState).toHaveBeenCalledWith({ x: 5, y: 6 }, true)
+    expect(el.tabIndex).toBe(-1)
+    expect(el.attrs.has('role')).toBe(false)
+    client.sendKeyEvent.mockClear()
+    made.keyboard!.press(66)
+    expect(client.sendKeyEvent).not.toHaveBeenCalled()
+  })
+
+  it('releases held keys when the page loses focus', () => {
+    const { G, made } = fakeGuacamole()
+    bindScreenInput(G, fakeElement(), client).set(true)
+    made.keyboard!.press(0xffe9)
+    window.dispatchEvent(new Event('blur'))
+    expect(client.sendKeyEvent).toHaveBeenLastCalledWith(0, 0xffe9)
+  })
+
+  it('gives the keyboard back on Control+Alt+Escape', () => {
+    const { G, made } = fakeGuacamole()
+    const el = fakeElement()
+    bindScreenInput(G, el, client).set(true)
+    made.keyboard!.press(0xffe3)
+    const event = Object.assign(new Event('keydown', { cancelable: true }), {
+      key: 'Escape',
+      ctrlKey: true,
+      altKey: true,
+    })
+    el.dispatchEvent(event)
+    expect(event.defaultPrevented).toBe(true)
+    expect(el.blur).toHaveBeenCalled()
+    expect(client.sendKeyEvent).toHaveBeenLastCalledWith(0, 0xffe3)
+  })
+})

mcp/src/ui/input.ts

@@ -0,0 +1,110 @@
+/**
+ * Mouse, touch and keyboard from the live screen to the computer, bound the
+ * way the portal Playground binds them (sai-platform
+ * `src/playground/guacamole/input.ts`). The keyboard listens on the screen
+ * element only, never the document, so the host's chat box keeps its keys.
+ *
+ * Starts off. `set(false)` first lets go of any key or button still held, so
+ * the computer is never left with a key down.
+ */
+
+export interface ScreenInput {
+  set: (on: boolean) => void
+  dispose: () => void
+}
+
+type MouseLike = {
+  onmousedown: ((s: Guacamole.Mouse.State) => void) | null
+  onmousemove: ((s: Guacamole.Mouse.State) => void) | null
+  onmouseup: ((s: Guacamole.Mouse.State) => void) | null
+}
+
+export function bindScreenInput(
+  G: typeof Guacamole,
+  element: HTMLElement,
+  client: Pick<Guacamole.Client, 'sendKeyEvent' | 'sendMouseState'>,
+): ScreenInput {
+  const keyboard = new G.Keyboard(element)
+  const pointers: MouseLike[] = [new G.Mouse(element), new G.Mouse.Touchscreen(element)]
+  let on = false
+  let last: Guacamole.Mouse.State = { x: 0, y: 0 }
+
+  const send = (state: Guacamole.Mouse.State): void => {
+    if (!on) return
+    last = { ...state }
+    // `true`: Guacamole maps the scaled screen box back to the computer's pixels.
+    client.sendMouseState(state, true)
+  }
+  for (const p of pointers) p.onmousedown = p.onmousemove = p.onmouseup = send
+  keyboard.onkeydown = (key: number) => {
+    if (!on) return true
+    client.sendKeyEvent(1, key)
+    return false
+  }
+  keyboard.onkeyup = (key: number) => {
+    if (on) client.sendKeyEvent(0, key)
+  }
+
+  const release = (): void => {
+    if (!on) return
+    keyboard.reset()
+    if (last.left || last.middle || last.right)
+      client.sendMouseState({ x: last.x, y: last.y }, true)
+    last = { x: last.x, y: last.y }
+  }
+  const focus = (): void => {
+    if (on) element.focus({ preventScroll: true })
+  }
+  // Every key goes to the computer while the screen has focus, Tab included:
+  // Ctrl+Alt+Esc hands the keyboard back to the page.
+  const exit = (event: KeyboardEvent): void => {
+    if (event.key !== 'Escape' || !event.ctrlKey || !event.altKey) return
+    event.preventDefault()
+    event.stopImmediatePropagation()
+    release()
+    element.blur()
+  }
+  const noMenu = (event: Event): void => {
+    if (on) event.preventDefault()
+  }
+  element.addEventListener('keydown', exit, true)
+  element.addEventListener('pointerdown', focus)
+  element.addEventListener('blur', release)
+  element.addEventListener('contextmenu', noMenu)
+  window.addEventListener('blur', release)
+
+  const set = (next: boolean): void => {
+    if (next === on) return
+    if (!next) release()
+    on = next
+    element.tabIndex = on ? 0 : -1
+    element.style.cursor = on ? '' : 'default'
+    element.style.touchAction = on ? 'none' : ''
+    if (on) {
+      element.setAttribute('role', 'application')
+      element.setAttribute(
+        'aria-label',
+        'Remote computer. Click to type. Press Control Alt Escape to return to the page.',
+      )
+    } else {
+      element.removeAttribute('role')
+      element.removeAttribute('aria-label')
+      element.blur()
+    }
+  }
+  set(false)
+
+  return {
+    set,
+    dispose: () => {
+      set(false)
+      keyboard.onkeydown = keyboard.onkeyup = null
+      for (const p of pointers) p.onmousedown = p.onmousemove = p.onmouseup = null
+      element.removeEventListener('keydown', exit, true)
+      element.removeEventListener('pointerdown', focus)
+      element.removeEventListener('blur', release)
+      element.removeEventListener('contextmenu', noMenu)
+      window.removeEventListener('blur', release)
+    },
+  }
+}

mcp/src/ui/live.ts

@@ -1,9 +1,10 @@
 /**
  * The task view's live screen: the vendored Guacamole client (same copy as the
- * portal Playground) over the tunnel `sai_view_live` hands out. View-only: no
- * mouse or keyboard is wired, so watching can't touch the computer.
+ * portal Playground) over the tunnel `sai_view_live` hands out. Input starts
+ * off; the task view turns it on only while Sai isn't working (`setInput`).
  */
 import './guacamole/all.js'
+import { bindScreenInput } from './input.js'
 
 /**
  * Why a stream dropped. `blocked`: the WebSocket never opened at all — the
@@ -24,6 +25,8 @@ function dropKind(code: number | undefined): DropKind {
 }
 
 export interface LiveHandle {
+  /** Forward the person's mouse and keyboard to the computer, or stop (releasing anything held). */
+  setInput: (on: boolean) => void
   /**
    * Disconnect. With `keepFrame`, leave the last frame on screen as an image
    * and return it (a JPEG data URL); nothing if no frame ever arrived.
@@ -47,6 +50,7 @@ export function connectLive(
   const display = client.getDisplay()
   const el = display.getElement()
   container.replaceChildren(el)
+  const input = bindScreenInput(G, el, client)
 
   const fit = (): void => {
     const w = display.getWidth()
@@ -85,7 +89,12 @@ export function connectLive(
   }
 
   return {
+    setInput: (on) => {
+      if (!closed) input.set(on)
+    },
     close: (keepFrame = false) => {
+      // Before anything else, while the client can still send the key-ups.
+      input.dispose()
       let frame: string | undefined
       // A display that never got a frame flattens to an empty canvas: no image then.
       if (keepFrame && display.getWidth() > 0 && display.getHeight() > 0) {

mcp/src/ui/task-view.html

@@ -196,7 +196,13 @@
         width: 100%;
         display: block;
       }
-      #screen-note {
+      /* The screen has the keyboard: show it, so the person knows where keys go. */
+      #screen > div:focus {
+        outline: 2px solid var(--ok);
+        outline-offset: -2px;
+      }
+      #screen-note,
+      #input-note {
         margin-top: 4px;
         color: var(--muted);
         font-size: 12px;
@@ -228,9 +234,14 @@
     <div id="screen-wrap" hidden>
       <div id="screen"></div>
       <div id="screen-note" hidden></div>
+      <div id="input-note" hidden></div>
       <button id="watch-here" hidden title="Take the live screen back to this view">
         Watch here
       </button>
+      <button id="use" hidden title="Connect to the computer and use its mouse and keyboard">
+        Use this computer
+      </button>
+      <button id="stop-use" hidden title="Disconnect and keep the last frame">Done</button>
     </div>
     <ol id="steps"></ol>
     <div id="steps-more"></div>