Load external plugin READMEs from source repositories

CONTRIBUTING.md

@@ -276,6 +276,33 @@ When a pull request updates `plugins/external.json` (for example, version update
    - `requires-submitter-fixes` when quality checks fail due to plugin issues
    - `awaiting-review` when checks cannot complete because of infrastructure/transient errors
 
+##### Website README
+
+External plugin detail pages load their README from GitHub during website data
+generation. No documentation copy or extra listing field is needed in this repo.
+The source comes from `plugins/external.json`: `source.repo`, `source.path`, and
+the revision in `source.sha` (preferred) or `source.ref`. Listings without either
+revision use the repository's default branch (`HEAD`).
+
+The build tries `README.md`, `readme.md`, and `Readme.md` in the plugin directory,
+then the repository root. Keep that README current upstream; pinned listings show
+documentation for the listed revision, so updating the listing refreshes its docs.
+Relative links and images resolve against the upstream README; paths beginning
+with a single `/` resolve from the repository root at the same revision.
+Remote Markdown uses the same HTML sanitizer as local documentation. JavaScript
+frontmatter is rejected before it can execute, and README bodies are excluded
+from the catalog page's client-side data.
+
+Downloads are limited to GitHub's raw-content host, six plugins at a time, with
+a ten-second timeout per request and a 512 KiB limit per README. Missing READMEs
+or download/rendering errors are logged and leave a source-repository link on
+the page instead of failing the entire website build.
+
+Run `node --test eng/lib/external-plugin-readme.test.mjs` to check retrieval.
+After `npm run website:build`, run
+`node --test website/scripts/external-plugin-readme.test.mjs` to check remote
+Markdown rendering, relative URLs, sanitization, and the generated plugin pages.
+
 ##### Maintainer review responsibilities
 
 Maintainers are responsible for confirming that the submission:

eng/generate-website-data.mjs

@@ -26,6 +26,7 @@ import {
   parseYamlFile,
 } from "./yaml-parser.mjs";
 import { readExternalPlugins } from "./external-plugin-validation.mjs";
+import { loadExternalPluginReadmes } from "./lib/external-plugin-readme.mjs";
 import {
   readExtensionPluginOwners,
   resolveExtensionPluginName,
@@ -1722,6 +1723,7 @@ async function main() {
     extensions,
   });
   const pluginsData = generatePluginsData(gitDates, resourceIndex);
+  await loadExternalPluginReadmes(pluginsData.items);
   const plugins = pluginsData.items;
   console.log(
     `✓ Generated ${plugins.length} plugins (${pluginsData.filters.tags.length} tags)`

eng/lib/external-plugin-readme.mjs

@@ -0,0 +1,93 @@
+const README_NAMES = ["README.md", "readme.md", "Readme.md"];
+const MAX_README_BYTES = 512 * 1024;
+
+/**
+ * Fetch documentation only from GitHub's raw-content host. Follow the listing's
+ * revision rather than silently showing documentation for a different release.
+ */
+export async function fetchExternalPluginReadme(
+  source,
+  { fetchImpl = fetch, warn = console.warn } = {},
+) {
+  if (source?.source !== "github") return null;
+  const { repo, path: pluginPath = "", sha, ref } = source;
+  const revision = sha || ref || "HEAD";
+  const folder = pluginPath === "/" ? "" : pluginPath;
+  try {
+    if (
+      typeof repo !== "string" ||
+      !/^[A-Za-z0-9_-][A-Za-z0-9._-]*\/[A-Za-z0-9_-][A-Za-z0-9._-]*$/.test(repo) ||
+      typeof revision !== "string" ||
+      !revision.trim() ||
+      revision.split("/").some((part) => part === "." || part === "..") ||
+      /[\x00-\x1f\x7f]/.test(revision) ||
+      typeof folder !== "string" ||
+      folder.includes("//") ||
+      folder.endsWith("/") ||
+      folder.split("/").some((part) => part === "." || part === "..") ||
+      /[\\?#]/.test(folder) ||
+      folder.startsWith("/")
+    ) {
+      throw new Error("Invalid GitHub source for external plugin README");
+    }
+
+    const locator = encodeURIComponent(revision);
+    const roots = folder ? [folder, ""] : [""];
+    for (const root of roots) {
+      for (const name of README_NAMES) {
+        const file = [...(root ? root.split("/") : []), name]
+          .map(encodeURIComponent).join("/");
+        const rawUrl = `https://raw.githubusercontent.com/${repo}/${locator}/${file}`;
+        const response = await fetchImpl(rawUrl, {
+          signal: AbortSignal.timeout(10_000),
+          redirect: "error",
+        });
+        if (response.status === 404) {
+          await response.body?.cancel();
+          continue;
+        }
+        if (!response.ok) {
+          await response.body?.cancel();
+          throw new Error(`HTTP ${response.status} for ${rawUrl}`);
+        }
+        if (Number(response.headers.get("content-length")) > MAX_README_BYTES) {
+          await response.body?.cancel();
+          throw new Error(`README exceeds ${MAX_README_BYTES} bytes: ${rawUrl}`);
+        }
+        const chunks = [];
+        let bytes = 0;
+        if (response.body) {
+          for await (const chunk of response.body) {
+            bytes += chunk.byteLength;
+            if (bytes > MAX_README_BYTES) {
+              throw new Error(`README exceeds ${MAX_README_BYTES} bytes: ${rawUrl}`);
+            }
+            chunks.push(chunk);
+          }
+        }
+        const markdown = Buffer.concat(chunks).toString("utf8");
+        if (!markdown.trim()) throw new Error(`Empty README: ${rawUrl}`);
+        return {
+          markdown,
+          sourceUrl: `https://github.com/${repo}/blob/${locator}/${file}`,
+          rawUrl,
+        };
+      }
+    }
+    warn(`[external-plugin-readme] No Markdown README found for ${repo} at ${revision}${folder ? ` (${folder})` : ""}`);
+  } catch (error) {
+    warn(`[external-plugin-readme] Could not load ${repo} at ${revision}: ${error.message}`);
+  }
+  return null;
+}
+
+export async function loadExternalPluginReadmes(plugins) {
+  const external = plugins.filter((plugin) => plugin.external);
+  let next = 0;
+  await Promise.all(Array.from({ length: Math.min(6, external.length) }, async () => {
+    while (next < external.length) {
+      const plugin = external[next++];
+      plugin.readme = await fetchExternalPluginReadme(plugin.source);
+    }
+  }));
+}

eng/lib/external-plugin-readme.test.mjs

@@ -0,0 +1,124 @@
+import assert from "node:assert/strict";
+import { test } from "node:test";
+import { fetchExternalPluginReadme, loadExternalPluginReadmes } from "./external-plugin-readme.mjs";
+
+const source = { source: "github", repo: "owner/repo", path: "plugins/example", ref: "v1.0.0" };
+
+function fixture(responses) {
+  const calls = [];
+  const warnings = [];
+  return {
+    calls,
+    warnings,
+    options: {
+      warn: (message) => warnings.push(message),
+      fetchImpl: async (url, options) => {
+        calls.push(url);
+        assert.equal(options.redirect, "error");
+        assert.ok(options.signal);
+        const response = responses.shift();
+        if (response instanceof Error) throw response;
+        return response ?? new Response("", { status: 404 });
+      },
+    },
+  };
+}
+
+test("infers the plugin README and prefers the listing SHA to its ref", async () => {
+  const f = fixture([new Response("# Plugin")]);
+  const result = await fetchExternalPluginReadme({ ...source, sha: "a".repeat(40) }, f.options);
+  assert.equal(result.markdown, "# Plugin");
+  assert.equal(result.rawUrl, `https://raw.githubusercontent.com/owner/repo/${"a".repeat(40)}/plugins/example/README.md`);
+  assert.equal(result.sourceUrl, `https://github.com/owner/repo/blob/${"a".repeat(40)}/plugins/example/README.md`);
+  assert.deepEqual(f.warnings, []);
+});
+
+test("uses a root path and HEAD for an unpinned listing", async () => {
+  const f = fixture([new Response("# Root")]);
+  await fetchExternalPluginReadme({ source: "github", repo: "owner/repo", path: "/" }, f.options);
+  assert.deepEqual(f.calls, ["https://raw.githubusercontent.com/owner/repo/HEAD/README.md"]);
+});
+
+test("encodes refs and path segments", async () => {
+  const f = fixture([new Response("# Plugin")]);
+  await fetchExternalPluginReadme({ ...source, path: "plugin docs", ref: "refs/tags/v1+docs" }, f.options);
+  assert.equal(f.calls[0], "https://raw.githubusercontent.com/owner/repo/refs%2Ftags%2Fv1%2Bdocs/plugin%20docs/README.md");
+});
+
+test("accepts ref-only package-style tags used by the catalog", async () => {
+  const f = fixture([new Response("# Radius")]);
+  const result = await fetchExternalPluginReadme({ ...source, ref: "radius@0.2.1" }, f.options);
+  assert.equal(result.markdown, "# Radius");
+  assert.match(f.calls[0], /\/radius%400\.2\.1\/plugins\/example\/README\.md$/);
+  assert.deepEqual(f.warnings, []);
+});
+
+test("tries common Markdown filename casing before the repository root", async () => {
+  const f = fixture([undefined, new Response("# Lowercase")]);
+  assert.equal((await fetchExternalPluginReadme(source, f.options)).markdown, "# Lowercase");
+  assert.match(f.calls[1], /plugins\/example\/readme\.md$/);
+  const root = fixture([undefined, undefined, undefined, new Response("# Root")]);
+  await fetchExternalPluginReadme(source, root.options);
+  assert.match(root.calls[3], /v1\.0\.0\/README\.md$/);
+});
+
+test("logs missing READMEs and retains a null fallback without changing revision", async () => {
+  const f = fixture([]);
+  assert.equal(await fetchExternalPluginReadme(source, f.options), null);
+  assert.equal(f.calls.length, 6);
+  assert.ok(f.calls.every((url) => url.includes("/v1.0.0/")));
+  assert.match(f.warnings[0], /No Markdown README found/);
+});
+
+test("logs HTTP, timeout, and network failures without trying another revision", async () => {
+  for (const response of [new Response("", { status: 403 }), new Response("", { status: 500 }), new Error("Timed out"), new Error("Network unavailable")]) {
+    const f = fixture([response]);
+    assert.equal(await fetchExternalPluginReadme(source, f.options), null);
+    assert.equal(f.calls.length, 1);
+    assert.match(f.warnings[0], /Could not load/);
+  }
+});
+
+test("limits declared and streamed response sizes and rejects empty files", async () => {
+  for (const response of [
+    new Response("large", { headers: { "content-length": String(512 * 1024 + 1) } }),
+    new Response("x".repeat(512 * 1024 + 1)),
+    new Response(" \n"),
+  ]) {
+    const f = fixture([response]);
+    assert.equal(await fetchExternalPluginReadme(source, f.options), null);
+    assert.equal(f.warnings.length, 1);
+  }
+});
+
+test("does not fetch non-GitHub sources or malformed GitHub paths", async () => {
+  const f = fixture([]);
+  assert.equal(await fetchExternalPluginReadme({ source: "git" }, f.options), null);
+  for (const overrides of [{ repo: "../bad" }, { repo: "host/repo/extra" }, { path: "../outside" }, { path: "/outside" }, { path: "a\\b" }, { path: null }, { path: "a//b" }, { ref: "../main" }, { ref: "." }]) {
+    assert.equal(await fetchExternalPluginReadme({ ...source, ...overrides }, f.options), null);
+  }
+  assert.deepEqual(f.calls, []);
+  assert.equal(f.warnings.length, 9);
+  assert.ok(f.warnings.every((message) => message.includes("Invalid GitHub source")));
+});
+
+test("loads only external plugins with bounded concurrency", async (t) => {
+  let active = 0;
+  let maximum = 0;
+  t.mock.method(globalThis, "fetch", async () => {
+    active++;
+    maximum = Math.max(maximum, active);
+    await new Promise((resolve) => setTimeout(resolve, 5));
+    active--;
+    return new Response("# Remote");
+  });
+  const plugins = [
+    { id: "built-in", readmeFile: "plugins/built-in/README.md" },
+    ...Array.from({ length: 14 }, () => ({ external: true, source })),
+  ];
+  await loadExternalPluginReadmes(plugins);
+  assert.equal(plugins[0].readme, undefined);
+  assert.equal(plugins[0].readmeFile, "plugins/built-in/README.md");
+  assert.ok(plugins.slice(1).every((plugin) => plugin.readme.markdown === "# Remote"));
+  assert.equal(maximum, 6);
+});