diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 95e2ac5fd6..61fa8d897f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -276,6 +276,33 @@ When a pull request updates `plugins/external.json` (for example, version update - `requires-submitter-fixes` when quality checks fail due to plugin issues - `awaiting-review` when checks cannot complete because of infrastructure/transient errors +##### Website README + +External plugin detail pages load their README from GitHub during website data +generation. No documentation copy or extra listing field is needed in this repo. +The source comes from `plugins/external.json`: `source.repo`, `source.path`, and +the revision in `source.sha` (preferred) or `source.ref`. Listings without either +revision use the repository's default branch (`HEAD`). + +The build tries `README.md`, `readme.md`, and `Readme.md` in the plugin directory, +then the repository root. Keep that README current upstream; pinned listings show +documentation for the listed revision, so updating the listing refreshes its docs. +Relative links and images resolve against the upstream README; paths beginning +with a single `/` resolve from the repository root at the same revision. +Remote Markdown uses the same HTML sanitizer as local documentation. JavaScript +frontmatter is rejected before it can execute, and README bodies are excluded +from the catalog page's client-side data. + +Downloads are limited to GitHub's raw-content host, six plugins at a time, with +a ten-second timeout per request and a 512 KiB limit per README. Missing READMEs +or download/rendering errors are logged and leave a source-repository link on +the page instead of failing the entire website build. + +Run `node --test eng/lib/external-plugin-readme.test.mjs` to check retrieval. +After `npm run website:build`, run +`node --test website/scripts/external-plugin-readme.test.mjs` to check remote +Markdown rendering, relative URLs, sanitization, and the generated plugin pages. + ##### Maintainer review responsibilities Maintainers are responsible for confirming that the submission: diff --git a/eng/generate-website-data.mjs b/eng/generate-website-data.mjs index d1d168381b..791d697234 100755 --- a/eng/generate-website-data.mjs +++ b/eng/generate-website-data.mjs @@ -26,6 +26,7 @@ import { parseYamlFile, } from "./yaml-parser.mjs"; import { readExternalPlugins } from "./external-plugin-validation.mjs"; +import { loadExternalPluginReadmes } from "./lib/external-plugin-readme.mjs"; import { readExtensionPluginOwners, resolveExtensionPluginName, @@ -1722,6 +1723,7 @@ async function main() { extensions, }); const pluginsData = generatePluginsData(gitDates, resourceIndex); + await loadExternalPluginReadmes(pluginsData.items); const plugins = pluginsData.items; console.log( `✓ Generated ${plugins.length} plugins (${pluginsData.filters.tags.length} tags)` diff --git a/eng/lib/external-plugin-readme.mjs b/eng/lib/external-plugin-readme.mjs new file mode 100644 index 0000000000..1f6b2b253b --- /dev/null +++ b/eng/lib/external-plugin-readme.mjs @@ -0,0 +1,93 @@ +const README_NAMES = ["README.md", "readme.md", "Readme.md"]; +const MAX_README_BYTES = 512 * 1024; + +/** + * Fetch documentation only from GitHub's raw-content host. Follow the listing's + * revision rather than silently showing documentation for a different release. + */ +export async function fetchExternalPluginReadme( + source, + { fetchImpl = fetch, warn = console.warn } = {}, +) { + if (source?.source !== "github") return null; + const { repo, path: pluginPath = "", sha, ref } = source; + const revision = sha || ref || "HEAD"; + const folder = pluginPath === "/" ? "" : pluginPath; + try { + if ( + typeof repo !== "string" || + !/^[A-Za-z0-9_-][A-Za-z0-9._-]*\/[A-Za-z0-9_-][A-Za-z0-9._-]*$/.test(repo) || + typeof revision !== "string" || + !revision.trim() || + revision.split("/").some((part) => part === "." || part === "..") || + /[\x00-\x1f\x7f]/.test(revision) || + typeof folder !== "string" || + folder.includes("//") || + folder.endsWith("/") || + folder.split("/").some((part) => part === "." || part === "..") || + /[\\?#]/.test(folder) || + folder.startsWith("/") + ) { + throw new Error("Invalid GitHub source for external plugin README"); + } + + const locator = encodeURIComponent(revision); + const roots = folder ? [folder, ""] : [""]; + for (const root of roots) { + for (const name of README_NAMES) { + const file = [...(root ? root.split("/") : []), name] + .map(encodeURIComponent).join("/"); + const rawUrl = `https://raw.githubusercontent.com/${repo}/${locator}/${file}`; + const response = await fetchImpl(rawUrl, { + signal: AbortSignal.timeout(10_000), + redirect: "error", + }); + if (response.status === 404) { + await response.body?.cancel(); + continue; + } + if (!response.ok) { + await response.body?.cancel(); + throw new Error(`HTTP ${response.status} for ${rawUrl}`); + } + if (Number(response.headers.get("content-length")) > MAX_README_BYTES) { + await response.body?.cancel(); + throw new Error(`README exceeds ${MAX_README_BYTES} bytes: ${rawUrl}`); + } + const chunks = []; + let bytes = 0; + if (response.body) { + for await (const chunk of response.body) { + bytes += chunk.byteLength; + if (bytes > MAX_README_BYTES) { + throw new Error(`README exceeds ${MAX_README_BYTES} bytes: ${rawUrl}`); + } + chunks.push(chunk); + } + } + const markdown = Buffer.concat(chunks).toString("utf8"); + if (!markdown.trim()) throw new Error(`Empty README: ${rawUrl}`); + return { + markdown, + sourceUrl: `https://github.com/${repo}/blob/${locator}/${file}`, + rawUrl, + }; + } + } + warn(`[external-plugin-readme] No Markdown README found for ${repo} at ${revision}${folder ? ` (${folder})` : ""}`); + } catch (error) { + warn(`[external-plugin-readme] Could not load ${repo} at ${revision}: ${error.message}`); + } + return null; +} + +export async function loadExternalPluginReadmes(plugins) { + const external = plugins.filter((plugin) => plugin.external); + let next = 0; + await Promise.all(Array.from({ length: Math.min(6, external.length) }, async () => { + while (next < external.length) { + const plugin = external[next++]; + plugin.readme = await fetchExternalPluginReadme(plugin.source); + } + })); +} diff --git a/eng/lib/external-plugin-readme.test.mjs b/eng/lib/external-plugin-readme.test.mjs new file mode 100644 index 0000000000..540e095adb --- /dev/null +++ b/eng/lib/external-plugin-readme.test.mjs @@ -0,0 +1,124 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { fetchExternalPluginReadme, loadExternalPluginReadmes } from "./external-plugin-readme.mjs"; + +const source = { source: "github", repo: "owner/repo", path: "plugins/example", ref: "v1.0.0" }; + +function fixture(responses) { + const calls = []; + const warnings = []; + return { + calls, + warnings, + options: { + warn: (message) => warnings.push(message), + fetchImpl: async (url, options) => { + calls.push(url); + assert.equal(options.redirect, "error"); + assert.ok(options.signal); + const response = responses.shift(); + if (response instanceof Error) throw response; + return response ?? new Response("", { status: 404 }); + }, + }, + }; +} + +test("infers the plugin README and prefers the listing SHA to its ref", async () => { + const f = fixture([new Response("# Plugin")]); + const result = await fetchExternalPluginReadme({ ...source, sha: "a".repeat(40) }, f.options); + assert.equal(result.markdown, "# Plugin"); + assert.equal(result.rawUrl, `https://raw.githubusercontent.com/owner/repo/${"a".repeat(40)}/plugins/example/README.md`); + assert.equal(result.sourceUrl, `https://github.com/owner/repo/blob/${"a".repeat(40)}/plugins/example/README.md`); + assert.deepEqual(f.warnings, []); +}); + +test("uses a root path and HEAD for an unpinned listing", async () => { + const f = fixture([new Response("# Root")]); + await fetchExternalPluginReadme({ source: "github", repo: "owner/repo", path: "/" }, f.options); + assert.deepEqual(f.calls, ["https://raw.githubusercontent.com/owner/repo/HEAD/README.md"]); +}); + +test("encodes refs and path segments", async () => { + const f = fixture([new Response("# Plugin")]); + await fetchExternalPluginReadme({ ...source, path: "plugin docs", ref: "refs/tags/v1+docs" }, f.options); + assert.equal(f.calls[0], "https://raw.githubusercontent.com/owner/repo/refs%2Ftags%2Fv1%2Bdocs/plugin%20docs/README.md"); +}); + +test("accepts ref-only package-style tags used by the catalog", async () => { + const f = fixture([new Response("# Radius")]); + const result = await fetchExternalPluginReadme({ ...source, ref: "radius@0.2.1" }, f.options); + assert.equal(result.markdown, "# Radius"); + assert.match(f.calls[0], /\/radius%400\.2\.1\/plugins\/example\/README\.md$/); + assert.deepEqual(f.warnings, []); +}); + +test("tries common Markdown filename casing before the repository root", async () => { + const f = fixture([undefined, new Response("# Lowercase")]); + assert.equal((await fetchExternalPluginReadme(source, f.options)).markdown, "# Lowercase"); + assert.match(f.calls[1], /plugins\/example\/readme\.md$/); + const root = fixture([undefined, undefined, undefined, new Response("# Root")]); + await fetchExternalPluginReadme(source, root.options); + assert.match(root.calls[3], /v1\.0\.0\/README\.md$/); +}); + +test("logs missing READMEs and retains a null fallback without changing revision", async () => { + const f = fixture([]); + assert.equal(await fetchExternalPluginReadme(source, f.options), null); + assert.equal(f.calls.length, 6); + assert.ok(f.calls.every((url) => url.includes("/v1.0.0/"))); + assert.match(f.warnings[0], /No Markdown README found/); +}); + +test("logs HTTP, timeout, and network failures without trying another revision", async () => { + for (const response of [new Response("", { status: 403 }), new Response("", { status: 500 }), new Error("Timed out"), new Error("Network unavailable")]) { + const f = fixture([response]); + assert.equal(await fetchExternalPluginReadme(source, f.options), null); + assert.equal(f.calls.length, 1); + assert.match(f.warnings[0], /Could not load/); + } +}); + +test("limits declared and streamed response sizes and rejects empty files", async () => { + for (const response of [ + new Response("large", { headers: { "content-length": String(512 * 1024 + 1) } }), + new Response("x".repeat(512 * 1024 + 1)), + new Response(" \n"), + ]) { + const f = fixture([response]); + assert.equal(await fetchExternalPluginReadme(source, f.options), null); + assert.equal(f.warnings.length, 1); + } +}); + +test("does not fetch non-GitHub sources or malformed GitHub paths", async () => { + const f = fixture([]); + assert.equal(await fetchExternalPluginReadme({ source: "git" }, f.options), null); + for (const overrides of [{ repo: "../bad" }, { repo: "host/repo/extra" }, { path: "../outside" }, { path: "/outside" }, { path: "a\\b" }, { path: null }, { path: "a//b" }, { ref: "../main" }, { ref: "." }]) { + assert.equal(await fetchExternalPluginReadme({ ...source, ...overrides }, f.options), null); + } + assert.deepEqual(f.calls, []); + assert.equal(f.warnings.length, 9); + assert.ok(f.warnings.every((message) => message.includes("Invalid GitHub source"))); +}); + +test("loads only external plugins with bounded concurrency", async (t) => { + let active = 0; + let maximum = 0; + t.mock.method(globalThis, "fetch", async () => { + active++; + maximum = Math.max(maximum, active); + await new Promise((resolve) => setTimeout(resolve, 5)); + active--; + return new Response("# Remote"); + }); + const plugins = [ + { id: "built-in", readmeFile: "plugins/built-in/README.md" }, + ...Array.from({ length: 14 }, () => ({ external: true, source })), + ]; + await loadExternalPluginReadmes(plugins); + assert.equal(plugins[0].readme, undefined); + assert.equal(plugins[0].readmeFile, "plugins/built-in/README.md"); + assert.ok(plugins.slice(1).every((plugin) => plugin.readme.markdown === "# Remote")); + assert.equal(maximum, 6); +}); diff --git a/website/scripts/external-plugin-readme.test.mjs b/website/scripts/external-plugin-readme.test.mjs new file mode 100644 index 0000000000..b5078aad0e --- /dev/null +++ b/website/scripts/external-plugin-readme.test.mjs @@ -0,0 +1,189 @@ +import assert from "node:assert/strict"; +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { after, test } from "node:test"; +import { createServer } from "vite"; + +const website = fileURLToPath(new URL("..", import.meta.url)); +const originalCwd = process.cwd(); +process.chdir(website); +const server = await createServer({ + root: website, + configFile: false, + server: { middlewareMode: true }, + appType: "custom", +}); +after(async () => { + await server.close(); + process.chdir(originalCwd); +}); +const { renderResourceMarkdown, readResourceMarkdown } = await server.ssrLoadModule("/src/lib/detail-page.ts"); +const { finalizePluginReadme } = await server.ssrLoadModule("/src/lib/plugin-readme.ts"); +const urls = { + linkBaseUrl: "https://github.com/owner/repo/blob/v1/plugins/example/README.md", + imageBaseUrl: "https://raw.githubusercontent.com/owner/repo/v1/plugins/example/README.md", +}; + +test("remote README uses the existing sanitizer, including raw HTML", () => { + const { markdownHtml } = renderResourceMarkdown(`# Plugin + +## Documentation + + + +Unsafe +Safe + + +
Overlay
+`, urls); + assert.match(markdownHtml, /Documentation/); + assert.doesNotMatch(markdownHtml, / { + const { markdownHtml } = renderResourceMarkdown( + '
Documentation
', + urls, + ); + assert.match(markdownHtml, /Documentation/); + assert.doesNotMatch(markdownHtml, /onmouseover/i); +}); + +test("sanitizes the final HTML after table-of-contents processing", () => { + const { markdownHtml } = renderResourceMarkdown( + '
Documentation
\n\n\n
Overlay
', + urls, + ); + const finalHtml = finalizePluginReadme(markdownHtml, urls); + assert.match(finalHtml, /Documentation/); + assert.doesNotMatch(finalHtml, /onmouseover| { + const warnings = []; + const markdownHtml = renderResourceMarkdown( + `
Documentation
`, + urls, + ).markdownHtml; + const finalHtml = finalizePluginReadme(markdownHtml, urls, (error) => warnings.push(error)); + assert.equal(finalHtml, ""); + assert.equal(warnings.length, 1); +}); + +test("resolves relative Markdown links, reference images, and raw HTML links upstream", () => { + const { markdownHtml } = renderResourceMarkdown(`[Guide](../../docs/guide.md) +[Section](#install) +![Image][image] + +[image]: ./assets/preview.png + +License + +[Root guide](/docs/guide.md) +[External](//example.com/docs) +![External image](//example.com/image.png) +`, urls); + assert.match(markdownHtml, /href="https:\/\/github.com\/owner\/repo\/blob\/v1\/docs\/guide.md"/); + assert.match(markdownHtml, /href="#install"/); + assert.match(markdownHtml, /src="https:\/\/raw.githubusercontent.com\/owner\/repo\/v1\/plugins\/example\/assets\/preview.png"/); + assert.match(markdownHtml, /href="https:\/\/github.com\/owner\/repo\/blob\/v1\/plugins\/example\/LICENSE"/); + assert.match(markdownHtml, /src="https:\/\/raw.githubusercontent.com\/owner\/repo\/v1\/assets\/logo.png"/); + assert.match(markdownHtml, /href="https:\/\/github.com\/owner\/repo\/blob\/v1\/docs\/guide.md"/); + assert.match(markdownHtml, /href="https:\/\/example.com\/docs"/); + assert.match(markdownHtml, /src="https:\/\/example.com\/image.png"/); +}); + +test("rejects both JavaScript frontmatter language names without executing code", () => { + for (const language of ["js", "javascript"]) { + const sentinel = `readmeFrontmatterExecuted_${language}`; + assert.equal(globalThis[sentinel], undefined); + assert.throws(() => renderResourceMarkdown( + `---${language}\n(globalThis.${sentinel} = true, {})\n---\n## Body`, + urls, + ), /JavaScript frontmatter is not supported/); + assert.equal(globalThis[sentinel], undefined); + } +}); + +test("keeps YAML and JSON frontmatter supported", () => { + for (const frontmatter of ["\ntitle: Plugin", 'json\n{"title":"Plugin"}']) { + const result = renderResourceMarkdown(`---${frontmatter}\n---\n## Body`, urls); + assert.match(result.markdownHtml, /Body/); + assert.doesNotMatch(result.markdownHtml, /title/); + } +}); + +test("root-relative URLs preserve an encoded tag and repository revision", () => { + const { markdownHtml } = renderResourceMarkdown("[Guide](/docs/guide.md)\n![Logo](/logo.png)", { + linkBaseUrl: urls.linkBaseUrl.replace("/v1/", "/refs%2Ftags%2Fradius%400.2.1/"), + imageBaseUrl: urls.imageBaseUrl.replace("/v1/", "/refs%2Ftags%2Fradius%400.2.1/"), + }); + assert.match(markdownHtml, /href="https:\/\/github.com\/owner\/repo\/blob\/refs%2Ftags%2Fradius%400\.2\.1\/docs\/guide.md"/); + assert.match(markdownHtml, /src="https:\/\/raw.githubusercontent.com\/owner\/repo\/refs%2Ftags%2Fradius%400\.2\.1\/logo.png"/); +}); + +test("catalog records omit README bodies while detail data retains them", async () => { + const { plugins } = await server.ssrLoadModule("/src/lib/site-data.ts"); + const data = JSON.parse(fs.readFileSync(path.join(website, "public", "data", "plugins.json"), "utf8")); + assert.ok(data.items.some((item) => item.readme?.markdown)); + assert.equal(plugins.length, data.items.length); + assert.ok(plugins.every((item) => !Object.hasOwn(item, "readme"))); +}); + +test("generated catalog does not send README bodies to the browser", () => { + const html = fs.readFileSync(path.join(website, "dist", "plugins", "index.html"), "utf8"); + const island = html.match(/]*component-export="PluginsCatalog"[^>]*>/)?.[0]; + assert.ok(island, "plugin catalog must be hydrated"); + assert.doesNotMatch(island, /"readme":/); +}); + +test("relative URL rewriting does not leak into local rendering", () => { + renderResourceMarkdown("[Remote](./guide.md)", urls); + assert.throws(() => renderResourceMarkdown("[Remote](./guide.md)", { + ...urls, + linkBaseUrl: "invalid URL", + })); + assert.match(renderResourceMarkdown("[Local](./guide.md)").markdownHtml, /href="\.\/guide.md"/); + assert.match( + renderResourceMarkdown('').markdownHtml, + /srcset="\.\/image-2x\.png 2x"/, + ); + assert.match(finalizePluginReadme('
Local
'), /style="color:red"|