fix(renderer): await worker startup and broker resource completion

tools/page-renderer/README.md

@@ -35,8 +35,11 @@ its own `max_chars`, payload budgets, and pagination.
 ## Supported pages and limits
 
 The helper supports scripts, GET/HEAD requests, and ordinary Web Workers. It
-waits for 500 ms without network activity, up to 10 seconds, before extracting
-the DOM. Content scheduled after that point may not be included. Rendering is
+waits for worker startup and all intercepted resources to finish, followed by
+500 ms without further resource activity, before extracting the DOM. A resource
+that starts during extraction causes the snapshot to be retried. This wait and
+extraction share a 10-second deadline. Workers need not terminate or send a
+message. Content scheduled after the quiet period may not be included. Rendering is
 limited to 15 seconds including browser startup, after the initial page
 download. Process and profile cleanup can take up to 8 additional seconds after
 cancellation or the deadline. A fresh browser is closed after each page, and no

tools/page-renderer/package.json

@@ -7,7 +7,7 @@
     "node": ">=20"
   },
   "scripts": {
-    "test": "node --test test.cjs",
+    "test": "node --test resource-monitor.test.cjs test.cjs",
     "install-browser": "playwright install firefox"
   },
   "dependencies": {

tools/page-renderer/render.cjs

@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
 const net = require('node:net');
 const readline = require('node:readline');
+const { ResourceMonitor } = require('./resource-monitor.cjs');
 
 async function main() {
   const { firefox } = require('playwright');
@@ -22,6 +23,7 @@ async function main() {
 
   const pending = new Map();
   let browser;
+  let resources;
   let nextId = 0;
   let fail;
   const failure = new Promise((_, reject) => { fail = reject; });
@@ -73,8 +75,10 @@ async function main() {
     });
     const context = await browser.newContext({ serviceWorkers: 'block', acceptDownloads: false });
     await context.routeWebSocket('**/*', socket => socket.close());
+    resources = new ResourceMonitor();
     let initial = true;
     await context.route('**/*', async route => {
+      const finished = resources.start();
       try {
         const request = route.request();
         if (initial && request.isNavigationRequest() && request.url() === initialUrl.href) {
@@ -97,21 +101,46 @@ async function main() {
       } catch (error) {
         fail(error);
         await route.abort().catch(() => {});
+      } finally {
+        finished();
       }
     });
     const page = await context.newPage();
+    page.on('worker', worker => {
+      const finished = resources.start();
+      let closed = false;
+      worker.once('close', () => { closed = true; finished(); });
+      // Let startup yield to a normal worker task before considering it idle.
+      void worker.evaluate(() => new Promise(resolve => {
+        const channel = new MessageChannel();
+        channel.port1.onmessage = () => {
+          channel.port1.close();
+          channel.port2.close();
+          resolve();
+        };
+        channel.port2.postMessage(null);
+      })).catch(error => {
+        if (!closed) fail(error);
+      }).finally(finished);
+    });
     const rendered = (async () => {
-      await page.goto(url, { waitUntil: 'networkidle', timeout: 10000 });
-      const output = await page.evaluate(limit => {
-        const value = document.documentElement.outerHTML;
-        return typeof value === 'string' && value.length <= limit ? value : null;
-      }, maxBytes);
+      await page.goto(url, { waitUntil: 'load', timeout: 10000 });
+      let version;
+      let output;
+      do {
+        version = await resources.wait();
+        output = await page.evaluate(limit => {
+          const value = document.documentElement.outerHTML;
+          return typeof value === 'string' && value.length <= limit ? value : null;
+        }, maxBytes);
+      } while (!resources.unchanged(version));
       if (output === null || Buffer.byteLength(output) > maxBytes) throw new Error('rendered HTML too large');
       return Buffer.from(output).toString('base64');
     })();
-    const body = await Promise.race([rendered, failure]);
+    const body = await Promise.race([rendered, failure, resources.deadline]);
     process.stdout.write(JSON.stringify({ type: 'html', body }) + '\n');
   } finally {
+    resources?.close();
     if (browser) await browser.close();
     await new Promise(resolve => denyProxy.close(resolve));
     input.removeListener('close', stop);

tools/page-renderer/resource-monitor.cjs

@@ -0,0 +1,71 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+class ResourceMonitor {
+  #pending = 0;
+  #version = 0;
+  #waiter;
+  #idleTimer;
+  #deadlineTimer;
+  #error;
+
+  constructor() {
+    this.deadline = new Promise((_, reject) => {
+      this.#deadlineTimer = setTimeout(() => {
+        const error = new Error('rendering exceeded 10000 ms');
+        this.#stop(error);
+        reject(error);
+      }, 10000);
+    });
+    this.deadline.catch(() => {});
+  }
+
+  start() {
+    this.#pending++;
+    this.#version++;
+    clearTimeout(this.#idleTimer);
+    let finished = false;
+    return () => {
+      if (finished) return;
+      finished = true;
+      this.#pending--;
+      this.#version++;
+      this.#schedule();
+    };
+  }
+
+  wait() {
+    if (this.#error) return Promise.reject(this.#error);
+    if (this.#waiter) throw new Error('resource idle wait already active');
+    return new Promise((resolve, reject) => {
+      this.#waiter = { resolve, reject };
+      this.#schedule();
+    });
+  }
+
+  unchanged(version) {
+    return !this.#error && this.#pending === 0 && this.#version === version;
+  }
+
+  close() {
+    clearTimeout(this.#deadlineTimer);
+    this.#stop(new Error('renderer stopped'));
+  }
+
+  #schedule() {
+    clearTimeout(this.#idleTimer);
+    if (!this.#waiter || this.#pending || this.#error) return;
+    this.#idleTimer = setTimeout(() => {
+      const { resolve } = this.#waiter;
+      this.#waiter = undefined;
+      resolve(this.#version);
+    }, 500);
+  }
+
+  #stop(error) {
+    clearTimeout(this.#idleTimer);
+    this.#error = error;
+    this.#waiter?.reject(error);
+    this.#waiter = undefined;
+  }
+}
+
+module.exports = { ResourceMonitor };

tools/page-renderer/resource-monitor.test.cjs

@@ -0,0 +1,108 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+const assert = require('node:assert/strict');
+const test = require('node:test');
+const { ResourceMonitor } = require('./resource-monitor.cjs');
+
+function clock(t) {
+  let now = 0;
+  const timers = new Set();
+  t.mock.method(global, 'setTimeout', (callback, delay) => {
+    const timer = { callback, at: now + delay };
+    timers.add(timer);
+    return timer;
+  });
+  t.mock.method(global, 'clearTimeout', timer => timers.delete(timer));
+  return milliseconds => {
+    const end = now + milliseconds;
+    while (true) {
+      const next = [...timers].sort((a, b) => a.at - b.at)[0];
+      if (!next || next.at > end) break;
+      now = next.at;
+      timers.delete(next);
+      next.callback();
+    }
+    now = end;
+  };
+}
+
+test('idle begins only after the last worker or resource finishes', async t => {
+  const advance = clock(t);
+  const resources = new ResourceMonitor();
+  t.after(() => resources.close());
+  const workerReady = resources.start();
+  const responseDelivered = resources.start();
+  let settled = false;
+  const idle = resources.wait().then(version => { settled = true; return version; });
+  workerReady();
+  advance(1000);
+  await Promise.resolve();
+  assert.equal(settled, false);
+  responseDelivered();
+  responseDelivered();
+  advance(499);
+  await Promise.resolve();
+  assert.equal(settled, false);
+  advance(1);
+  assert.equal(resources.unchanged(await idle), true);
+});
+
+test('activity resets the quiet period and invalidates an earlier snapshot', async t => {
+  const advance = clock(t);
+  const resources = new ResourceMonitor();
+  t.after(() => resources.close());
+  let settled = false;
+  const idle = resources.wait().then(version => { settled = true; return version; });
+  advance(400);
+  const finished = resources.start();
+  finished();
+  advance(100);
+  await Promise.resolve();
+  assert.equal(settled, false);
+  advance(400);
+  const version = await idle;
+  const next = resources.start();
+  assert.equal(resources.unchanged(version), false);
+  next();
+  assert.equal(resources.unchanged(version), false);
+  const updated = resources.wait();
+  advance(500);
+  assert.equal(resources.unchanged(await updated), true);
+});
+
+test('a stalled resource reaches the shared render deadline', async t => {
+  const advance = clock(t);
+  const resources = new ResourceMonitor();
+  t.after(() => resources.close());
+  resources.start();
+  const idle = assert.rejects(resources.wait(), /rendering exceeded 10000 ms/);
+  const deadline = assert.rejects(resources.deadline, /rendering exceeded 10000 ms/);
+  advance(10000);
+  await Promise.all([idle, deadline]);
+  await assert.rejects(resources.wait(), /rendering exceeded 10000 ms/);
+});
+
+test('the deadline also covers work after resources become idle', async t => {
+  const advance = clock(t);
+  const resources = new ResourceMonitor();
+  t.after(() => resources.close());
+  const idle = resources.wait();
+  advance(500);
+  const version = await idle;
+  const deadline = assert.rejects(resources.deadline, /rendering exceeded 10000 ms/);
+  advance(9500);
+  await deadline;
+  assert.equal(resources.unchanged(version), false);
+});
+
+test('shutdown cancels an idle wait and clears its deadline', async t => {
+  const advance = clock(t);
+  const resources = new ResourceMonitor();
+  const idle = assert.rejects(resources.wait(), /renderer stopped/);
+  let deadlineReached = false;
+  resources.deadline.catch(() => { deadlineReached = true; });
+  resources.close();
+  advance(10000);
+  await idle;
+  assert.equal(deadlineReached, false);
+  await assert.rejects(resources.wait(), /renderer stopped/);
+});

tools/page-renderer/test.cjs

@@ -20,15 +20,17 @@ function run(html, fetch, options = {}) {
     const timer = setTimeout(() => {
       child.kill('SIGKILL');
       reject(new Error('renderer timed out: ' + diagnostics));
-    }, 15000);
+    }, options.timeoutMs || 15000);
     child.on('error', reject);
-    lines.on('line', line => {
+    lines.on('line', async line => {
       try {
         const message = JSON.parse(line);
         if (message.type === 'fetch') {
           requests.push(message);
-          const reply = fetch(message);
-          child.stdin.write(JSON.stringify({ id: message.id, ...reply }) + '\n');
+          const reply = await fetch(message);
+          if (!child.stdin.destroyed && !child.stdin.writableEnded) {
+            child.stdin.write(JSON.stringify({ id: message.id, ...reply }) + '\n');
+          }
         } else result = message;
       } catch (error) {
         child.kill('SIGKILL');
@@ -79,6 +81,57 @@ test('renders external scripts, fetches, and worker content beyond 8000 characte
   assert.equal(result.requests.length, 4);
 });
 
+test('waits for worker startup, its response, and the resulting page fetch', async () => {
+  const resources = {
+    '/worker.js': response(`
+      const until = performance.now() + 900;
+      while (performance.now() < until) {}
+      fetch('/worker.json').then(r => r.json()).then(v => postMessage(v.text));
+    `),
+    '/worker.json': response(JSON.stringify({ text: 'Worker text' }), 'application/json'),
+    '/details.txt': response('with details', 'text/plain'),
+  };
+  const { done } = run(`<main></main><script>
+    const worker = new Worker('/worker.js');
+    worker.onmessage = async e => {
+      const response = await fetch('/details.txt');
+      document.querySelector('main').textContent = e.data + ' ' + await response.text();
+    };
+  </script>`, async request => {
+    const pathname = new URL(request.url).pathname;
+    if (pathname === '/worker.json') await new Promise(resolve => setTimeout(resolve, 750));
+    return resources[pathname] || { error: 'unexpected resource' };
+  });
+  const result = await done;
+  assert.equal(result.type, 'html', result.message);
+  assert.equal(result.code, 0);
+  assert.ok(Buffer.from(result.body, 'base64').toString().includes('<main>Worker text with details</main>'));
+  assert.deepEqual(result.requests.map(request => new URL(request.url).pathname), [
+    '/worker.js', '/worker.json', '/details.txt',
+  ]);
+});
+
+test('idle and self-closing workers need not send a message', async () => {
+  for (const script of ['onmessage = event => postMessage(event.data);', 'close();']) {
+    const { done } = run('<main>ready</main><script>new Worker("/worker.js")</script>', () => response(script));
+    const result = await done;
+    assert.equal(result.type, 'html', result.message);
+    assert.ok(Buffer.from(result.body, 'base64').toString().includes('<main>ready</main>'));
+  }
+});
+
+test('a worker resource that never receives a broker reply reaches the deadline', async () => {
+  const { done } = run('<script>new Worker("/worker.js")</script>', request => {
+    if (new URL(request.url).pathname === '/worker.js') return response('fetch("/waiting.json");');
+    return new Promise(() => {});
+  }, { timeoutMs: 25000 });
+  const result = await done;
+  assert.equal(result.type, 'error');
+  assert.equal(result.code, 1);
+  assert.match(result.message, /rendering exceeded 10000 ms/);
+  assert.ok(result.requests.some(request => new URL(request.url).pathname === '/waiting.json'));
+});
+
 test('broker failures and redirected resources fail the whole render', async () => {
   for (const reply of [{ error: 'blocked destination' }, { status: 302, headers: { location: 'http://127.0.0.1/' }, body: '' }]) {
     const { done } = run('<main>placeholder</main><script src="/app.js"></script>', () => reply);