diff --git a/tools/page-renderer/README.md b/tools/page-renderer/README.md
index 79e36f7..8650731 100644
--- a/tools/page-renderer/README.md
+++ b/tools/page-renderer/README.md
@@ -35,8 +35,11 @@ its own `max_chars`, payload budgets, and pagination.
## Supported pages and limits
The helper supports scripts, GET/HEAD requests, and ordinary Web Workers. It
-waits for 500 ms without network activity, up to 10 seconds, before extracting
-the DOM. Content scheduled after that point may not be included. Rendering is
+waits for worker startup and all intercepted resources to finish, followed by
+500 ms without further resource activity, before extracting the DOM. A resource
+that starts during extraction causes the snapshot to be retried. This wait and
+extraction share a 10-second deadline. Workers need not terminate or send a
+message. Content scheduled after the quiet period may not be included. Rendering is
limited to 15 seconds including browser startup, after the initial page
download. Process and profile cleanup can take up to 8 additional seconds after
cancellation or the deadline. A fresh browser is closed after each page, and no
diff --git a/tools/page-renderer/package.json b/tools/page-renderer/package.json
index a4b9298..87f103a 100644
--- a/tools/page-renderer/package.json
+++ b/tools/page-renderer/package.json
@@ -7,7 +7,7 @@
"node": ">=20"
},
"scripts": {
- "test": "node --test test.cjs",
+ "test": "node --test resource-monitor.test.cjs test.cjs",
"install-browser": "playwright install firefox"
},
"dependencies": {
diff --git a/tools/page-renderer/render.cjs b/tools/page-renderer/render.cjs
index 5de84cc..e9c65cc 100644
--- a/tools/page-renderer/render.cjs
+++ b/tools/page-renderer/render.cjs
@@ -1,6 +1,7 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
const net = require('node:net');
const readline = require('node:readline');
+const { ResourceMonitor } = require('./resource-monitor.cjs');
async function main() {
const { firefox } = require('playwright');
@@ -22,6 +23,7 @@ async function main() {
const pending = new Map();
let browser;
+ let resources;
let nextId = 0;
let fail;
const failure = new Promise((_, reject) => { fail = reject; });
@@ -73,8 +75,10 @@ async function main() {
});
const context = await browser.newContext({ serviceWorkers: 'block', acceptDownloads: false });
await context.routeWebSocket('**/*', socket => socket.close());
+ resources = new ResourceMonitor();
let initial = true;
await context.route('**/*', async route => {
+ const finished = resources.start();
try {
const request = route.request();
if (initial && request.isNavigationRequest() && request.url() === initialUrl.href) {
@@ -97,21 +101,46 @@ async function main() {
} catch (error) {
fail(error);
await route.abort().catch(() => {});
+ } finally {
+ finished();
}
});
const page = await context.newPage();
+ page.on('worker', worker => {
+ const finished = resources.start();
+ let closed = false;
+ worker.once('close', () => { closed = true; finished(); });
+ // Let startup yield to a normal worker task before considering it idle.
+ void worker.evaluate(() => new Promise(resolve => {
+ const channel = new MessageChannel();
+ channel.port1.onmessage = () => {
+ channel.port1.close();
+ channel.port2.close();
+ resolve();
+ };
+ channel.port2.postMessage(null);
+ })).catch(error => {
+ if (!closed) fail(error);
+ }).finally(finished);
+ });
const rendered = (async () => {
- await page.goto(url, { waitUntil: 'networkidle', timeout: 10000 });
- const output = await page.evaluate(limit => {
- const value = document.documentElement.outerHTML;
- return typeof value === 'string' && value.length <= limit ? value : null;
- }, maxBytes);
+ await page.goto(url, { waitUntil: 'load', timeout: 10000 });
+ let version;
+ let output;
+ do {
+ version = await resources.wait();
+ output = await page.evaluate(limit => {
+ const value = document.documentElement.outerHTML;
+ return typeof value === 'string' && value.length <= limit ? value : null;
+ }, maxBytes);
+ } while (!resources.unchanged(version));
if (output === null || Buffer.byteLength(output) > maxBytes) throw new Error('rendered HTML too large');
return Buffer.from(output).toString('base64');
})();
- const body = await Promise.race([rendered, failure]);
+ const body = await Promise.race([rendered, failure, resources.deadline]);
process.stdout.write(JSON.stringify({ type: 'html', body }) + '\n');
} finally {
+ resources?.close();
if (browser) await browser.close();
await new Promise(resolve => denyProxy.close(resolve));
input.removeListener('close', stop);
diff --git a/tools/page-renderer/resource-monitor.cjs b/tools/page-renderer/resource-monitor.cjs
new file mode 100644
index 0000000..e114731
--- /dev/null
+++ b/tools/page-renderer/resource-monitor.cjs
@@ -0,0 +1,71 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+class ResourceMonitor {
+ #pending = 0;
+ #version = 0;
+ #waiter;
+ #idleTimer;
+ #deadlineTimer;
+ #error;
+
+ constructor() {
+ this.deadline = new Promise((_, reject) => {
+ this.#deadlineTimer = setTimeout(() => {
+ const error = new Error('rendering exceeded 10000 ms');
+ this.#stop(error);
+ reject(error);
+ }, 10000);
+ });
+ this.deadline.catch(() => {});
+ }
+
+ start() {
+ this.#pending++;
+ this.#version++;
+ clearTimeout(this.#idleTimer);
+ let finished = false;
+ return () => {
+ if (finished) return;
+ finished = true;
+ this.#pending--;
+ this.#version++;
+ this.#schedule();
+ };
+ }
+
+ wait() {
+ if (this.#error) return Promise.reject(this.#error);
+ if (this.#waiter) throw new Error('resource idle wait already active');
+ return new Promise((resolve, reject) => {
+ this.#waiter = { resolve, reject };
+ this.#schedule();
+ });
+ }
+
+ unchanged(version) {
+ return !this.#error && this.#pending === 0 && this.#version === version;
+ }
+
+ close() {
+ clearTimeout(this.#deadlineTimer);
+ this.#stop(new Error('renderer stopped'));
+ }
+
+ #schedule() {
+ clearTimeout(this.#idleTimer);
+ if (!this.#waiter || this.#pending || this.#error) return;
+ this.#idleTimer = setTimeout(() => {
+ const { resolve } = this.#waiter;
+ this.#waiter = undefined;
+ resolve(this.#version);
+ }, 500);
+ }
+
+ #stop(error) {
+ clearTimeout(this.#idleTimer);
+ this.#error = error;
+ this.#waiter?.reject(error);
+ this.#waiter = undefined;
+ }
+}
+
+module.exports = { ResourceMonitor };
diff --git a/tools/page-renderer/resource-monitor.test.cjs b/tools/page-renderer/resource-monitor.test.cjs
new file mode 100644
index 0000000..76599a3
--- /dev/null
+++ b/tools/page-renderer/resource-monitor.test.cjs
@@ -0,0 +1,108 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+const assert = require('node:assert/strict');
+const test = require('node:test');
+const { ResourceMonitor } = require('./resource-monitor.cjs');
+
+function clock(t) {
+ let now = 0;
+ const timers = new Set();
+ t.mock.method(global, 'setTimeout', (callback, delay) => {
+ const timer = { callback, at: now + delay };
+ timers.add(timer);
+ return timer;
+ });
+ t.mock.method(global, 'clearTimeout', timer => timers.delete(timer));
+ return milliseconds => {
+ const end = now + milliseconds;
+ while (true) {
+ const next = [...timers].sort((a, b) => a.at - b.at)[0];
+ if (!next || next.at > end) break;
+ now = next.at;
+ timers.delete(next);
+ next.callback();
+ }
+ now = end;
+ };
+}
+
+test('idle begins only after the last worker or resource finishes', async t => {
+ const advance = clock(t);
+ const resources = new ResourceMonitor();
+ t.after(() => resources.close());
+ const workerReady = resources.start();
+ const responseDelivered = resources.start();
+ let settled = false;
+ const idle = resources.wait().then(version => { settled = true; return version; });
+ workerReady();
+ advance(1000);
+ await Promise.resolve();
+ assert.equal(settled, false);
+ responseDelivered();
+ responseDelivered();
+ advance(499);
+ await Promise.resolve();
+ assert.equal(settled, false);
+ advance(1);
+ assert.equal(resources.unchanged(await idle), true);
+});
+
+test('activity resets the quiet period and invalidates an earlier snapshot', async t => {
+ const advance = clock(t);
+ const resources = new ResourceMonitor();
+ t.after(() => resources.close());
+ let settled = false;
+ const idle = resources.wait().then(version => { settled = true; return version; });
+ advance(400);
+ const finished = resources.start();
+ finished();
+ advance(100);
+ await Promise.resolve();
+ assert.equal(settled, false);
+ advance(400);
+ const version = await idle;
+ const next = resources.start();
+ assert.equal(resources.unchanged(version), false);
+ next();
+ assert.equal(resources.unchanged(version), false);
+ const updated = resources.wait();
+ advance(500);
+ assert.equal(resources.unchanged(await updated), true);
+});
+
+test('a stalled resource reaches the shared render deadline', async t => {
+ const advance = clock(t);
+ const resources = new ResourceMonitor();
+ t.after(() => resources.close());
+ resources.start();
+ const idle = assert.rejects(resources.wait(), /rendering exceeded 10000 ms/);
+ const deadline = assert.rejects(resources.deadline, /rendering exceeded 10000 ms/);
+ advance(10000);
+ await Promise.all([idle, deadline]);
+ await assert.rejects(resources.wait(), /rendering exceeded 10000 ms/);
+});
+
+test('the deadline also covers work after resources become idle', async t => {
+ const advance = clock(t);
+ const resources = new ResourceMonitor();
+ t.after(() => resources.close());
+ const idle = resources.wait();
+ advance(500);
+ const version = await idle;
+ const deadline = assert.rejects(resources.deadline, /rendering exceeded 10000 ms/);
+ advance(9500);
+ await deadline;
+ assert.equal(resources.unchanged(version), false);
+});
+
+test('shutdown cancels an idle wait and clears its deadline', async t => {
+ const advance = clock(t);
+ const resources = new ResourceMonitor();
+ const idle = assert.rejects(resources.wait(), /renderer stopped/);
+ let deadlineReached = false;
+ resources.deadline.catch(() => { deadlineReached = true; });
+ resources.close();
+ advance(10000);
+ await idle;
+ assert.equal(deadlineReached, false);
+ await assert.rejects(resources.wait(), /renderer stopped/);
+});
diff --git a/tools/page-renderer/test.cjs b/tools/page-renderer/test.cjs
index b8f9486..4d6637c 100644
--- a/tools/page-renderer/test.cjs
+++ b/tools/page-renderer/test.cjs
@@ -20,15 +20,17 @@ function run(html, fetch, options = {}) {
const timer = setTimeout(() => {
child.kill('SIGKILL');
reject(new Error('renderer timed out: ' + diagnostics));
- }, 15000);
+ }, options.timeoutMs || 15000);
child.on('error', reject);
- lines.on('line', line => {
+ lines.on('line', async line => {
try {
const message = JSON.parse(line);
if (message.type === 'fetch') {
requests.push(message);
- const reply = fetch(message);
- child.stdin.write(JSON.stringify({ id: message.id, ...reply }) + '\n');
+ const reply = await fetch(message);
+ if (!child.stdin.destroyed && !child.stdin.writableEnded) {
+ child.stdin.write(JSON.stringify({ id: message.id, ...reply }) + '\n');
+ }
} else result = message;
} catch (error) {
child.kill('SIGKILL');
@@ -79,6 +81,57 @@ test('renders external scripts, fetches, and worker content beyond 8000 characte
assert.equal(result.requests.length, 4);
});
+test('waits for worker startup, its response, and the resulting page fetch', async () => {
+ const resources = {
+ '/worker.js': response(`
+ const until = performance.now() + 900;
+ while (performance.now() < until) {}
+ fetch('/worker.json').then(r => r.json()).then(v => postMessage(v.text));
+ `),
+ '/worker.json': response(JSON.stringify({ text: 'Worker text' }), 'application/json'),
+ '/details.txt': response('with details', 'text/plain'),
+ };
+ const { done } = run(``, async request => {
+ const pathname = new URL(request.url).pathname;
+ if (pathname === '/worker.json') await new Promise(resolve => setTimeout(resolve, 750));
+ return resources[pathname] || { error: 'unexpected resource' };
+ });
+ const result = await done;
+ assert.equal(result.type, 'html', result.message);
+ assert.equal(result.code, 0);
+ assert.ok(Buffer.from(result.body, 'base64').toString().includes('Worker text with details'));
+ assert.deepEqual(result.requests.map(request => new URL(request.url).pathname), [
+ '/worker.js', '/worker.json', '/details.txt',
+ ]);
+});
+
+test('idle and self-closing workers need not send a message', async () => {
+ for (const script of ['onmessage = event => postMessage(event.data);', 'close();']) {
+ const { done } = run('ready', () => response(script));
+ const result = await done;
+ assert.equal(result.type, 'html', result.message);
+ assert.ok(Buffer.from(result.body, 'base64').toString().includes('ready'));
+ }
+});
+
+test('a worker resource that never receives a broker reply reaches the deadline', async () => {
+ const { done } = run('', request => {
+ if (new URL(request.url).pathname === '/worker.js') return response('fetch("/waiting.json");');
+ return new Promise(() => {});
+ }, { timeoutMs: 25000 });
+ const result = await done;
+ assert.equal(result.type, 'error');
+ assert.equal(result.code, 1);
+ assert.match(result.message, /rendering exceeded 10000 ms/);
+ assert.ok(result.requests.some(request => new URL(request.url).pathname === '/waiting.json'));
+});
+
test('broker failures and redirected resources fail the whole render', async () => {
for (const reply of [{ error: 'blocked destination' }, { status: 302, headers: { location: 'http://127.0.0.1/' }, body: '' }]) {
const { done } = run('placeholder', () => reply);