diff --git a/package-lock.json b/package-lock.json index 2c7c934a..3adaddfb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -24,7 +24,7 @@ "youtube-transcript": "^1.3.0" }, "devDependencies": { - "@playwright/test": "^1.54.0", + "@playwright/test": "^1.64.1", "@sveltejs/vite-plugin-svelte": "^5.0.3", "@vitest/coverage-v8": "^3.2.4", "@vitest/ui": "^3.2.4", @@ -792,19 +792,19 @@ } }, "node_modules/@playwright/test": { - "version": "1.59.1", - "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.59.1.tgz", - "integrity": "sha512-PG6q63nQg5c9rIi4/Z5lR5IVF7yU5MqmKaPOe0HSc0O2cX1fPi96sUQu5j7eo4gKCkB2AnNGoWt7y4/Xx3Kcqg==", + "version": "1.64.1", + "resolved": "https://registry.npmmirror.com/@playwright/test/-/test-1.64.1.tgz", + "integrity": "sha512-IaYGml/nSOiZWROfPtuIL7neuu9qDOZ3TNvNljUIrhDMxiGTgPjzj/g9p9RxpS2tSepPDRY+UwAaEeiKtUvIAg==", "dev": true, "license": "Apache-2.0", "dependencies": { - "playwright": "1.59.1" + "playwright": "1.64.1" }, "bin": { "playwright": "cli.js" }, "engines": { - "node": ">=18" + "node": ">=20" } }, "node_modules/@polka/url": { @@ -2998,50 +2998,32 @@ } }, "node_modules/playwright": { - "version": "1.59.1", - "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.59.1.tgz", - "integrity": "sha512-C8oWjPR3F81yljW9o5OxcWzfh6avkVwDD2VYdwIGqTkl+OGFISgypqzfu7dOe4QNLL2aqcWBmI3PMtLIK233lw==", + "version": "1.64.1", + "resolved": "https://registry.npmmirror.com/playwright/-/playwright-1.64.1.tgz", + "integrity": "sha512-HCrVhHr5QQuwYoswEUGTb7gGp522yflbv/ST8nF4sFWGuLjQCt17Q/EubpGGSNKYn1tsTCPZxqHhHCfVURu7UA==", "dev": true, "license": "Apache-2.0", "dependencies": { - "playwright-core": "1.59.1" + "playwright-core": "1.64.1" }, "bin": { "playwright": "cli.js" }, "engines": { - "node": ">=18" - }, - "optionalDependencies": { - "fsevents": "2.3.2" + "node": ">=20" } }, "node_modules/playwright-core": { - "version": "1.59.1", - "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.59.1.tgz", - "integrity": "sha512-HBV/RJg81z5BiiZ9yPzIiClYV/QMsDCKUyogwH9p3MCP6IYjUFu/MActgYAvK0oWyV9NlwM3GLBjADyWgydVyg==", + "version": "1.64.1", + "resolved": "https://registry.npmmirror.com/playwright-core/-/playwright-core-1.64.1.tgz", + "integrity": "sha512-BC4s8y6SEVA6Q1rSxZTto8hI1KUI3xXlHGZgvGqP4IqPdI/a6FX9H29IDNz7vKo9l21b6LfYosScPL6M3xxyDQ==", "dev": true, "license": "Apache-2.0", "bin": { "playwright-core": "cli.js" }, "engines": { - "node": ">=18" - } - }, - "node_modules/playwright/node_modules/fsevents": { - "version": "2.3.2", - "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", - "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", - "dev": true, - "hasInstallScript": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + "node": ">=20" } }, "node_modules/postcss": { @@ -4791,4 +4773,4 @@ "license": "MIT" } } -} \ No newline at end of file +} diff --git a/package.json b/package.json index cfd5889c..370b90b4 100644 --- a/package.json +++ b/package.json @@ -28,7 +28,7 @@ "test:ci": "npm run test:ci:web && npm run test:ci:android" }, "devDependencies": { - "@playwright/test": "^1.54.0", + "@playwright/test": "^1.64.1", "@sveltejs/vite-plugin-svelte": "^5.0.3", "@vitest/coverage-v8": "^3.2.4", "@vitest/ui": "^3.2.4", @@ -54,4 +54,4 @@ "xlsx": "^0.18.5", "youtube-transcript": "^1.3.0" } -} \ No newline at end of file +} diff --git a/src/content/bridge.js b/src/content/bridge.js index 1118cf41..139192a7 100644 --- a/src/content/bridge.js +++ b/src/content/bridge.js @@ -116,6 +116,28 @@ export function setupBridgeEvents() { }; window.addEventListener("bds:assistant-message-id", handlers["bds:assistant-message-id"]); + // The completion stream hands over the reply text before the renderer touches + // it. Keep the freshest copy per session so tag attributes can be parsed from + // the bytes the model actually wrote, not from the lossy DOM. + handlers["bds:raw-reply"] = (event) => { + let detail = event.detail; + if (typeof detail === "string") { + try { detail = JSON.parse(detail); } catch (e) { return; } + } + const rawText = detail?.rawText; + const sessionId = detail?.sessionId; + if (typeof rawText !== "string" || !rawText || !sessionId) return; + + state.rawRepliesBySession.set(String(sessionId), rawText); + // Only the session on screen is ever consulted; keep the map from growing. + while (state.rawRepliesBySession.size > 5) { + const oldest = state.rawRepliesBySession.keys().next().value; + state.rawRepliesBySession.delete(oldest); + } + devLog("Parser", `stream supplied verbatim reply text for ${sessionId}`); + }; + window.addEventListener("bds:raw-reply", handlers["bds:raw-reply"]); + // The MAIN world could not fetch the history at all (no token, http error, // network failure). Surface it where the voice diagnostics can see it. handlers["bds:history-msgs-error"] = (event) => { diff --git a/src/content/dom/message-text-katex-payload.test.js b/src/content/dom/message-text-katex-payload.test.js new file mode 100644 index 00000000..2314c063 --- /dev/null +++ b/src/content/dom/message-text-katex-payload.test.js @@ -0,0 +1,63 @@ +// @vitest-environment jsdom + +import { describe, expect, it } from "vitest"; +import { extractMessageMarkdown } from "./message-text.js"; + +/** + * A BDS tag attribute that merely contains two dollar characters is rendered + * by the page as inline math, so the DOM path sees a KaTeX subtree instead of + * the payload. The extractor used to return an empty string for a KaTeX + * subtree with no LaTeX annotation, which deleted the payload from the + * extracted text. + */ + +function messageNode(build) { + const node = document.createElement("div"); + const md = document.createElement("div"); + md.className = "ds-markdown"; + build(md); + node.appendChild(md); + return node; +} + +function katexSubtree(tex) { + const span = document.createElement("span"); + span.className = "katex"; + const math = document.createElement("math"); + const annotation = document.createElement("annotation"); + annotation.setAttribute("encoding", "application/x-tex"); + annotation.textContent = tex; + math.appendChild(annotation); + span.appendChild(math); + return span; +} + +describe("KaTeX extraction never deletes payload content", () => { + it("recovers the tex source from the annotation", () => { + const text = extractMessageMarkdown( + messageNode((md) => { + md.appendChild(document.createTextNode("X")); + md.appendChild(katexSubtree("Y")); + md.appendChild(document.createTextNode("Z")); + }), + ); + + expect(text).toContain("Y"); + expect(text).toContain("X"); + expect(text).toContain("Z"); + }); + + it("never collapses a bare KaTeX span to nothing", () => { + const text = extractMessageMarkdown( + messageNode(function (md) { + const span = document.createElement("span"); + span.className = "katex"; + span.textContent = "survives"; + md.appendChild(span); + }), + ); + + expect(text.trim()).not.toBe(""); + expect(text).toContain("survives"); + }); +}); \ No newline at end of file diff --git a/src/content/dom/message-text.js b/src/content/dom/message-text.js index c34c44bd..b0f7a935 100644 --- a/src/content/dom/message-text.js +++ b/src/content/dom/message-text.js @@ -80,10 +80,16 @@ function katexToMarkdown(el) { return el.classList.contains("katex-display") ? `\n$$${tex}$$\n` : `$${tex}$`; } - // No LaTeX source to recover: emit the MathML token text as plain text rather - // than wrapping a garbled formula in math delimiters. + // No LaTeX source to recover. Emitting "" here deleted the whole subtree from + // the extracted text, so a BDS tag attribute that merely *contained* two `$` + // characters lost everything between them with no error anywhere (the KaTeX + // rendering of an unparseable formula is a red error span, not a formula, and + // the payload was never meant to be math). Never return empty: fall back to + // the MathML token text, then to the element text, so the payload at least + // survives as text instead of vanishing. const math = el.querySelector("math"); - return math ? String(math.textContent || "").trim() : ""; + if (math) return String(math.textContent || "").trim(); + return String(el.textContent || "").trim(); } /** Marker kept so a diagram-only message is not dropped as empty content. */ diff --git a/src/content/message-processor.svelte.js b/src/content/message-processor.svelte.js index 0ba516bd..98a87da3 100644 --- a/src/content/message-processor.svelte.js +++ b/src/content/message-processor.svelte.js @@ -747,8 +747,20 @@ export function processMessageNode(node, nodeIndex = -1, nodes = null, context = }, Math.max(0, SETTLE_STALL_MS - timeSinceUpdate) + SETTLE_RECHECK_MARGIN_MS); } + // The completion stream can hand over the reply text verbatim, which is the + // only copy that still contains escapes the Markdown renderer ate (`\/` -> + // `/`). It arrives after the stream ends, so it must take part in the hash or + // the settled message would never be re-parsed with the better source. + const sessionId = getCurrentConversationIdInline(); + const streamedRawText = isLatestAssistant && sessionId + ? state.rawRepliesBySession?.get(sessionId) || null + : null; + const parseSource = streamedRawText || rawText; + // Include settlement state in hash so transition to 'finished' triggers a final re-parse - const signature = simpleHash(rawText + (isSettled ? ":settled" : ":streaming")); + const signature = simpleHash( + parseSource + (isSettled ? ":settled" : ":streaming") + (streamedRawText ? ":verbatim" : ""), + ); const shouldForceCloseTags = isSettled && isStalled; if (stateData.hash === signature && stateData.forceClosedTags === shouldForceCloseTags) { @@ -761,7 +773,29 @@ export function processMessageNode(node, nodeIndex = -1, nodes = null, context = stateData.hash = signature; stateData.forceClosedTags = shouldForceCloseTags; - const parsed = parseBdsMessage(richText, shouldForceCloseTags); + // Parse twice, by intent: + // + // - TAGS AND ATTRIBUTES come from rawText. Attribute values are + // opaque payloads, but richText represents any `$tex$ $ span as the + // page's KaTeX DOM subtree, so a `$s='hi'; $s` payload arrives as HTML + // and corrupts the arguments. rawText collapses KaTeX back to + // `$tex$` (see katexToMarkdown), restoring the delimiters. + // + // - VISIBLE TEXT comes from richText. The overlay renders it with + // `marked.parse()`, which has no KaTeX plugin — it only passes the + // renderer's own `` markup through. Feeding the + // overlay rawText would show literal `$tex$` instead of the formula. + // + // This fixes the `$...$` deletion bug on its own, and — when the completion + // stream handed over a verbatim copy — `\/` -> `/` as well. Bare URL -> + // autolink and `&` -> `&` are still rewritten by the Markdown renderer + // before the DOM exists; they are covered by link-artifacts and HTML entity + // decoding respectively. + const parsed = parseBdsMessage(parseSource, shouldForceCloseTags); + if (parsed.visibleText !== undefined) { + // Keep the rendered KaTeX markup for the overlay only. + parsed.visibleText = parseBdsMessage(richText, shouldForceCloseTags).visibleText; + } const preGateBlocks = parsed.renderableBlocks; // --- BIDI DIRECTION (issue #181) --- diff --git a/src/content/state.js b/src/content/state.js index 50f91a99..b0a0e47c 100644 --- a/src/content/state.js +++ b/src/content/state.js @@ -55,6 +55,8 @@ const state = { chatMessagesBySession: new Map(), /** @type {Map} id of the reply the completion stream just produced, keyed by session ID — the same id DeepSeek's own read-aloud button uses */ assistantMessageIds: new Map(), + /** @type {Map} verbatim reply text from the completion stream, keyed by session ID. The rendered DOM has already lost escapes like `\\/`, so BDS tag attributes must be parsed from this copy when it is available. */ + rawRepliesBySession: new Map(), /** Token price tracking — session-level totals */ pricing: { /** @type {string|null} current model being used */ diff --git a/src/injected/fetch-patch.js b/src/injected/fetch-patch.js index a7bc1652..15271b02 100644 --- a/src/injected/fetch-patch.js +++ b/src/injected/fetch-patch.js @@ -1,10 +1,12 @@ import { mutatePayload } from "./payload-mutator.js"; import { + currentSessionIdFromUrl, emitAssistantMessageId, findMessageId, findMessageIdInSse, isReplyStreamUrl, } from "./message-id.js"; +import { extractReplyTextFromSse } from "./reply-text.js"; /** * Patch window.fetch to intercept chat completion requests. @@ -285,6 +287,24 @@ async function parseSSEUsage(response, modelName, captureMessageId) { if (!captureMessageId) return; const messageId = findMessageIdInSse(buffer); if (messageId) emitAssistantMessageId(messageId); + + // Hand the untouched reply text to the content script. The DOM this reply + // becomes is lossy — CommonMark has already eaten escapes like `\\/` by the + // time it renders — so tag attributes must be parsed from this copy. + const { content } = extractReplyTextFromSse(buffer); + if (content) emitRawReply(content, messageId); +} + +/** Tell the content script which reply text this stream produced, verbatim. */ +function emitRawReply(rawText, messageId) { + window.dispatchEvent(new CustomEvent("bds:raw-reply", { + detail: JSON.stringify({ + messageId: messageId || null, + sessionId: currentSessionIdFromUrl(), + rawText, + timestamp: Date.now(), + }), + })); } function emitTokenUsage(inputTokens, outputTokens, modelName) { diff --git a/src/injected/reply-text.js b/src/injected/reply-text.js new file mode 100644 index 00000000..35408cb8 --- /dev/null +++ b/src/injected/reply-text.js @@ -0,0 +1,73 @@ +/** + * Rebuild a streamed reply from its raw SSE body. + * + * The DOM is a lossy copy of what the model wrote: by the time a reply is + * rendered, CommonMark has already eaten escapes like `\\/`, and KaTeX has + * rewritten anything between two `$`. Reading the stream is the only place the + * original bytes still exist. + * + * DeepSeek streams the reply as a series of APPEND operations targeting + * `response/fragments/-1/content`, after an opening frame that carries the + * message id and the first fragment's text. + */ + +/** The field every streamed content chunk appends to. */ +const CONTENT_PATH = "response/fragments/-1/content"; + +/** + * Pull the reply text and message id out of a raw SSE response body. + * + * @param {string} text the full response body, already decoded + * @returns {{ messageId: string|null, content: string }} + */ +export function extractReplyTextFromSse(text) { + const source = String(text || ""); + let messageId = null; + const parts = []; + + for (const line of source.split("\n")) { + const trimmed = line.trim(); + if (!trimmed.startsWith("data: ")) continue; + + const payload = trimmed.slice(6).trim(); + if (!payload || payload === "[DONE]") continue; + + let chunk; + try { + chunk = JSON.parse(payload); + } catch { + // Comments, heartbeats and other non-JSON frames carry no reply text. + continue; + } + if (!chunk || typeof chunk !== "object") continue; + + if (!messageId) { + const found = findResponseMessageId(chunk); + if (found) messageId = found; + } + + // Two shapes carry reply text: the opening frame's fragment list, and the + // incremental APPEND operations that follow it. + if (chunk.p === CONTENT_PATH && typeof chunk.v === "string") { + parts.push(chunk.v); + continue; + } + + const fragment = chunk.v?.response?.fragments?.[0]?.content; + if (typeof fragment === "string") { + parts.push(fragment); + } + } + + return { messageId, content: parts.join("") }; +} + +/** The id of the reply this stream is producing, if the frame reports it. */ +function findResponseMessageId(chunk) { + const response = chunk.v?.response; + if (!response) return null; + const id = response.message_id ?? response.msg_id ?? null; + if (id === null || id === undefined) return null; + const normalised = String(id).trim(); + return normalised || null; +} diff --git a/src/injected/reply-text.test.js b/src/injected/reply-text.test.js new file mode 100644 index 00000000..00c2c6f3 --- /dev/null +++ b/src/injected/reply-text.test.js @@ -0,0 +1,71 @@ +import { describe, expect, it } from "vitest"; +import { extractReplyTextFromSse } from "./reply-text.js"; + +/** Join raw SSE event strings into one response body. */ +const sse = (...events) => events.join(""); + +const append = (v) => + `data: ${JSON.stringify({ p: "response/fragments/-1/content", o: "APPEND", v })}\n\n`; + +describe("extractReplyTextFromSse", () => { + it("stitches the streamed APPEND chunks back into the full reply", () => { + const text = sse(append("He"), append("llo")); + expect(extractReplyTextFromSse(text).content).toBe("Hello"); + }); + + it("includes the first fragment carried by the opening frame", () => { + // DeepSeek sends the opening character inside fragments[0].content, then + // appends the rest. Missing it would drop the first character of every + // reply. + const opening = `data: ${JSON.stringify({ + v: { + response: { + message_id: 2, + fragments: [{ id: 2, type: "RESPONSE", content: "<" }], + }, + }, + })}\n\n`; + const text = sse(opening, append("BDS")); + expect(extractReplyTextFromSse(text).content).toBe(" { + // The whole point of reading the stream: `\\/` survives here but becomes + // `/` once the reply goes through marked. Losing it in this layer would + // leave the corruption unfixable downstream. + const text = sse(append("D:"), append("\\"), append("/"), append("test")); + expect(extractReplyTextFromSse(text).content).toBe("D:\\/test"); + }); + + it("reports the message id the stream opened with", () => { + const opening = `data: ${JSON.stringify({ + v: { response: { message_id: 2, fragments: [] } }, + })}\n\n`; + expect(extractReplyTextFromSse(opening).messageId).toBe("2"); + }); + + it("ignores APPEND chunks aimed at other fields", () => { + const other = `data: ${JSON.stringify({ + p: "response/status", + o: "SET", + v: "FINISHED", + })}\n\n`; + const text = sse(other, append("ok")); + expect(extractReplyTextFromSse(text).content).toBe("ok"); + }); + + it("tolerates non-JSON and sentinel lines", () => { + const text = sse( + "event: ready\n", + "data: not json\n\n", + "data: [DONE]\n\n", + append("ok"), + ); + expect(extractReplyTextFromSse(text).content).toBe("ok"); + }); + + it("returns empty results for an empty stream", () => { + expect(extractReplyTextFromSse("")).toEqual({ messageId: null, content: "" }); + expect(extractReplyTextFromSse(null)).toEqual({ messageId: null, content: "" }); + }); +});