fix(desktop): snapshot background preview tabs without timing out

apps/desktop/src/preview/DesktopBrowserHost.test.ts

@@ -30,24 +30,34 @@ const command = (
     }),
   );
 
-const makeDebuggee = () => {
+const makeDebuggee = (prepareSurface: () => Promise<unknown> = async () => true) => {
   const emitter = new NodeEvents.EventEmitter();
-  const pending: Array<() => void> = [];
+  const pending: Array<(cause?: Error) => void> = [];
   const sent: Array<unknown> = [];
   const sendWaiters: Array<() => void> = [];
   const debuggee = Object.assign(emitter, {
     sendCommand: (method: string, params?: { id?: number }) =>
-      new Promise((resolve) => {
+      new Promise((resolve, reject) => {
         if (method === "Target.getTargetInfo") {
           resolve({ targetInfo: { targetId: "GUEST" } });
           return;
         }
         sent.push(params?.id);
-        pending.push(() => resolve({ method }));
+        pending.push((cause) => (cause ? reject(cause) : resolve({ method })));
         sendWaiters.shift()?.();
       }),
   });
+  const surfaceRequests: string[] = [];
   const webContents = Object.assign(new NodeEvents.EventEmitter(), {
+    id: 42,
+    isDestroyed: () => false,
+    hostWebContents: {
+      isDestroyed: () => false,
+      executeJavaScript: (expression: string) => {
+        surfaceRequests.push(expression);
+        return surfaceRequests.length === 1 ? prepareSurface() : Promise.resolve(true);
+      },
+    },
     getURL: () => "http://localhost/",
     getTitle: () => "Page",
     getUserAgent: () => "Electron",
@@ -61,9 +71,10 @@ const makeDebuggee = () => {
     navigate: () => webContents.emit("did-navigate"),
     /** Ids of the commands the debugger received, in order. */
     sent,
+    surfaceRequests,
     nextSend: () => new Promise<void>((resolve) => sendWaiters.push(resolve)),
     release: () => pending.splice(0).forEach((resolve) => resolve()),
-    releaseFirst: () => pending.shift()?.(),
+    releaseFirst: (cause?: Error) => pending.shift()?.(cause),
   };
 };
 
@@ -74,6 +85,16 @@ const takeEvents = (host: DesktopBrowserHost.DesktopBrowserHost["Service"], coun
     Effect.map((lines) => lines.map((line) => decodeEvent(new TextDecoder().decode(line)))),
   );
 
+const sendScreenshot = Effect.fn("test.sendScreenshot")(function* (
+  host: DesktopBrowserHost.DesktopBrowserHost["Service"],
+  debuggee: ReturnType<typeof makeDebuggee>,
+  id: number,
+) {
+  const sent = debuggee.nextSend();
+  yield* command(host, id, "Page.captureScreenshot");
+  yield* Effect.promise(() => sent);
+});
+
 /**
  * Ids of the replies the host sends, in order. A screenshot's reply leaves after its turn
  * has passed on, so by then the next waiting screenshot, if any, has been sent. Subscribe
@@ -94,6 +115,63 @@ const watchReplies = (host: DesktopBrowserHost.DesktopBrowserHost["Service"]) =>
   });
 
 describe("DesktopBrowserHost", () => {
+  it.effect.each([false, true])(
+    "waits for a renderable surface and restores it after capture (failed: %s)",
+    (failed) =>
+      Effect.gen(function* () {
+        const host = yield* DesktopBrowserHost.make;
+        const ready = Promise.withResolvers<boolean>();
+        const debuggee = makeDebuggee(() => ready.promise);
+        host.attach(key, debuggee.tab);
+        const replies = yield* watchReplies(host);
+
+        yield* command(host, 1, "Page.captureScreenshot");
+        expect(debuggee.sent).toEqual([]);
+
+        const sent = debuggee.nextSend();
+        ready.resolve(true);
+        yield* Effect.promise(() => sent);
+        debuggee.releaseFirst(failed ? new Error("Capture failed") : undefined);
+        expect(yield* Queue.take(replies)).toBe(1);
+        expect(debuggee.surfaceRequests).toHaveLength(2);
+      }).pipe(Effect.scoped),
+  );
+
+  it.effect("does not send a screenshot superseded while its surface was becoming ready", () =>
+    Effect.gen(function* () {
+      const host = yield* DesktopBrowserHost.make;
+      const ready = Promise.withResolvers<boolean>();
+      const debuggee = makeDebuggee(() => ready.promise);
+      host.attach(key, debuggee.tab);
+      const replies = yield* watchReplies(host);
+
+      yield* command(host, 1, "Page.captureScreenshot");
+      debuggee.navigate();
+      ready.resolve(true);
+      expect(yield* Queue.take(replies)).toBe(1);
+      expect(debuggee.sent).toEqual([]);
+      expect(debuggee.surfaceRequests).toHaveLength(2);
+    }).pipe(Effect.scoped),
+  );
+
+  it.effect.each<[string, () => Promise<unknown>]>([
+    ["never becomes ready", async () => false],
+    ["can't be reached", () => Promise.reject(new Error("Script failed"))],
+  ])("still issues a screenshot when its render surface %s", ([, prepareSurface]) =>
+    Effect.gen(function* () {
+      const host = yield* DesktopBrowserHost.make;
+      const debuggee = makeDebuggee(prepareSurface);
+      host.attach(key, debuggee.tab);
+      const replies = yield* watchReplies(host);
+
+      yield* sendScreenshot(host, debuggee, 1);
+      debuggee.releaseFirst();
+      expect(yield* Queue.take(replies)).toBe(1);
+      expect(debuggee.sent).toEqual([1]);
+      expect(debuggee.surfaceRequests).toHaveLength(2);
+    }).pipe(Effect.scoped),
+  );
+
   it.effect("announces tabs already attached to a backend that starts later", () =>
     Effect.gen(function* () {
       const host = yield* DesktopBrowserHost.make;
@@ -120,7 +198,7 @@ describe("DesktopBrowserHost", () => {
             message: encodeJson({ id, method, sessionId: "t3-preview-page" }),
           }),
         );
-      yield* command(1, "Page.captureScreenshot");
+      yield* sendScreenshot(host, debuggee, 1);
       yield* host.handleCommandLine(encodeJson({ type: "release", ...key }));
       yield* command(2, "DOM.enable");
       debuggee.release();
@@ -138,13 +216,15 @@ describe("DesktopBrowserHost", () => {
       host.attach(key, debuggee.tab);
       const replies = yield* watchReplies(host);
 
-      yield* command(host, 1, "Page.captureScreenshot");
+      yield* sendScreenshot(host, debuggee, 1);
       yield* command(host, 2, "Page.captureScreenshot");
       yield* command(host, 3, "DOM.enable");
       expect(debuggee.sent).toEqual([1, 3]);
 
+      const secondSent = debuggee.nextSend();
       debuggee.releaseFirst();
       expect(yield* Queue.take(replies)).toBe(1);
+      yield* Effect.promise(() => secondSent);
       expect(debuggee.sent).toEqual([1, 3, 2]);
     }).pipe(Effect.scoped),
   );
@@ -156,19 +236,23 @@ describe("DesktopBrowserHost", () => {
       host.attach(key, debuggee.tab);
       const replies = yield* watchReplies(host);
 
-      yield* command(host, 1, "Page.captureScreenshot");
+      yield* sendScreenshot(host, debuggee, 1);
       yield* command(host, 2, "Page.captureScreenshot");
       yield* command(host, 3, "Page.captureScreenshot");
+      const secondSent = debuggee.nextSend();
       debuggee.navigate();
+      yield* Effect.promise(() => secondSent);
       expect(debuggee.sent).toEqual([1, 2]);
 
       // The superseded screenshot answering late must not start a third beside the second.
       debuggee.releaseFirst();
       expect(yield* Queue.take(replies)).toBe(1);
       expect(debuggee.sent).toEqual([1, 2]);
 
+      const thirdSent = debuggee.nextSend();
       debuggee.releaseFirst();
       expect(yield* Queue.take(replies)).toBe(2);
+      yield* Effect.promise(() => thirdSent);
       expect(debuggee.sent).toEqual([1, 2, 3]);
     }).pipe(Effect.scoped),
   );
@@ -179,7 +263,7 @@ describe("DesktopBrowserHost", () => {
       const debuggee = makeDebuggee();
       host.attach(key, debuggee.tab);
 
-      yield* command(host, 1, "Page.captureScreenshot");
+      yield* sendScreenshot(host, debuggee, 1);
       yield* command(host, 2, "Page.captureScreenshot");
       yield* host.handleCommandLine(encodeJson({ type: "release", ...key }));
       yield* command(host, 3, "Page.captureScreenshot");

apps/desktop/src/preview/DesktopBrowserHost.ts

@@ -4,12 +4,14 @@
  * the relay shares it.
  */
 import {
+  DESKTOP_PREVIEW_CAPTURE_SURFACE_TRIGGER,
   DesktopBrowserCommand,
   DesktopBrowserEvent,
   type DesktopBrowserEvent as DesktopBrowserEventType,
 } from "@t3tools/contracts";
 import * as Context from "effect/Context";
 import * as NodePath from "node:path";
+import * as NodeCrypto from "node:crypto";
 import * as Effect from "effect/Effect";
 import * as Layer from "effect/Layer";
 import * as Option from "effect/Option";
@@ -112,7 +114,7 @@ export const make = Effect.gen(function* () {
         }
         const turn = {};
         tab.screenshotTurn = turn;
-        capture()
+        withCaptureSurface(tab, relay, turn, capture)
           .finally(() => {
             if (tab.screenshotTurn === turn) nextScreenshot(tab);
           })
@@ -122,6 +124,35 @@ export const make = Effect.gen(function* () {
       else tab.waitingScreenshots.push(start);
     });
 
+  const withCaptureSurface = async (
+    tab: AttachedTab,
+    relay: CdpRelayConnection,
+    turn: object,
+    capture: () => Promise<unknown>,
+  ) => {
+    const wc = tab.debuggee.webContents;
+    const host = wc.hostWebContents;
+    if (!host || host.isDestroyed() || wc.isDestroyed()) return capture();
+    const captureId = NodeCrypto.randomUUID();
+    const setSurface = (active: boolean) =>
+      host.executeJavaScript(
+        `globalThis[${JSON.stringify(DESKTOP_PREVIEW_CAPTURE_SURFACE_TRIGGER)}]?.(${wc.id}, ${JSON.stringify(captureId)}, ${active})`,
+      );
+    try {
+      // The guest stays offscreen until React commits the activity lease and Electron
+      // receives its new bounds. CDP can otherwise wait forever for a composited frame.
+      // A window that isn't painting, like a minimized one, never reports ready, yet its
+      // on-screen tab can still be captured, so the screenshot is sent either way.
+      await setSurface(true).catch(() => false);
+      if (tab.relay !== relay || tab.screenshotTurn !== turn || wc.isDestroyed()) {
+        throw new Error("The preview connection that asked for this screenshot closed.");
+      }
+      return await capture();
+    } finally {
+      if (!host.isDestroyed()) await setSurface(false).catch(() => undefined);
+    }
+  };
+
   const relayFor = (tab: AttachedTab) => {
     if (tab.relay) return tab.relay;
     const { webContents, debugger: debuggee } = tab.debuggee;

apps/web/src/browser/ElectronBrowserHost.tsx

@@ -19,13 +19,15 @@ import { useAtomCommand } from "~/state/use-atom-command";
 
 import { readPreviewAnnotationTheme } from "./annotationTheme";
 import { useBrowserDefaults } from "./browserDefaults";
+import { installBrowserCaptureSurface } from "./browserCaptureSurface";
 import { useBrowserPointerStore } from "./browserPointerStore";
 import { HostedBrowserWebview } from "./HostedBrowserWebview";
 import { openUrlInPreview } from "./openFileInPreview";
 import { rendersServerTabNatively } from "./previewRuntime";
 import { previewRuntimeTabId } from "./previewRuntimeTabId";
 
 export function ElectronBrowserHost() {
+  useEffect(() => (isElectron ? installBrowserCaptureSurface() : undefined), []);
   const previewByThreadKey = useActivePreviewSessions();
   const primaryEnvironmentId = useAtomValue(primaryEnvironmentIdAtom);
   useDesktopHostedPreviewSessions(primaryEnvironmentId);

apps/web/src/browser/browserCaptureSurface.test.ts

@@ -0,0 +1,87 @@
+import { DESKTOP_PREVIEW_CAPTURE_SURFACE_TRIGGER } from "@t3tools/contracts";
+import { expect, it, vi } from "vite-plus/test";
+
+import { installBrowserCaptureSurface } from "./browserCaptureSurface";
+import { acquireBrowserSurfaceActivity, useBrowserSurfaceStore } from "./browserSurfaceStore";
+
+function setup() {
+  vi.useFakeTimers();
+  const frames = new Map<number, FrameRequestCallback>();
+  let sequence = 0;
+  vi.stubGlobal("window", {
+    requestAnimationFrame: (callback: FrameRequestCallback) => {
+      frames.set(++sequence, callback);
+      return sequence;
+    },
+    cancelAnimationFrame: (id: number) => frames.delete(id),
+    setTimeout: globalThis.setTimeout,
+    clearTimeout: globalThis.clearTimeout,
+  });
+  vi.stubGlobal("document", {
+    querySelectorAll: () =>
+      [42, 43].map((id) => ({
+        getWebContentsId: () => id,
+        dataset: { previewTab: `tab-${id}` },
+        closest: () => ({ getBoundingClientRect: () => ({ width: 1280, height: 800 }) }),
+      })),
+  });
+  useBrowserSurfaceStore.setState({ activityByTabId: {}, byTabId: {} });
+  const dispose = installBrowserCaptureSurface();
+  const setSurface = Reflect.get(globalThis, DESKTOP_PREVIEW_CAPTURE_SURFACE_TRIGGER) as (
+    webContentsId: number,
+    captureId: string,
+    active: boolean,
+  ) => Promise<boolean>;
+  return {
+    setSurface,
+    dispose,
+    paint: () => {
+      const callbacks = [...frames.values()];
+      frames.clear();
+      for (const callback of callbacks) callback(0);
+    },
+    [Symbol.dispose]() {
+      dispose();
+      vi.unstubAllGlobals();
+      vi.useRealTimers();
+    },
+  };
+}
+
+it("activates only the captured tab and preserves another consumer when capture ends", async () => {
+  using surface = setup();
+  const releaseRecording = acquireBrowserSurfaceActivity("tab-42");
+  const ready = surface.setSurface(42, "capture-1", true);
+  expect(useBrowserSurfaceStore.getState().activityByTabId).toEqual({ "tab-42": 2 });
+  surface.paint();
+  surface.paint();
+  expect(await ready).toBe(true);
+
+  await surface.setSurface(42, "capture-1", false);
+  expect(useBrowserSurfaceStore.getState().activityByTabId).toEqual({ "tab-42": 1 });
+  releaseRecording();
+  expect(useBrowserSurfaceStore.getState().activityByTabId).toEqual({});
+});
+
+it("releases a tab whose embedder never paints", async () => {
+  using surface = setup();
+  const ready = surface.setSurface(42, "capture-1", true);
+  await vi.advanceTimersByTimeAsync(1_000);
+  expect(await ready).toBe(false);
+  expect(useBrowserSurfaceStore.getState().activityByTabId).toEqual({});
+});
+
+it("bounds rendering for an abandoned capture and releases pending work on teardown", async () => {
+  using surface = setup();
+  const ready = surface.setSurface(42, "capture-1", true);
+  surface.paint();
+  surface.paint();
+  expect(await ready).toBe(true);
+  await vi.advanceTimersByTimeAsync(10_000);
+  expect(useBrowserSurfaceStore.getState().activityByTabId).toEqual({});
+
+  const pending = surface.setSurface(43, "capture-2", true);
+  surface.dispose();
+  expect(await pending).toBe(false);
+  expect(useBrowserSurfaceStore.getState().activityByTabId).toEqual({});
+});