diff --git a/package-lock.json b/package-lock.json index 377843c3..74d8a8f8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,6 +16,7 @@ "@fontsource/teko": "^5.1.1", "@tailwindcss/vite": "^4.1.4", "astro": "^7.0.0", + "mdast-util-to-markdown": "^2.1.2", "sharp": "^0.35.0", "starlight-github-alerts": "^0.5.0", "starlight-image-zoom": "^0.16.0", diff --git a/package.json b/package.json index a4afe443..eaa930df 100644 --- a/package.json +++ b/package.json @@ -23,6 +23,7 @@ "@fontsource/teko": "^5.1.1", "@tailwindcss/vite": "^4.1.4", "astro": "^7.0.0", + "mdast-util-to-markdown": "^2.1.2", "sharp": "^0.35.0", "starlight-github-alerts": "^0.5.0", "starlight-image-zoom": "^0.16.0", diff --git a/src/build/integration.ts b/src/build/integration.ts index e9733f55..51d9be60 100644 --- a/src/build/integration.ts +++ b/src/build/integration.ts @@ -16,11 +16,13 @@ import { rmSync, writeFileSync, } from 'fs'; -import { basename, dirname, join } from 'path'; +import { basename, dirname, join, resolve } from 'path'; import { fileURLToPath } from 'url'; import { discoverAllVersions, minorKey, parseOverrides, readProductsJson } from './versions'; import { computeSlugRename, toTitleCase } from './dirRename'; import { rewriteLinks } from './linkRewrite'; +import { createRawLinkRewriter } from './rawLinkRewrite'; +import { latestProductVersion } from '../versionUtils'; import { VERSION_SLUG_RE } from './cleanupDirs'; import { bustAstroCache, @@ -343,6 +345,34 @@ export async function main(): Promise { } } + // Normalize copied source too: raw LLM exports bypass the Remark pipeline. + const sections = (config: DocsConfig) => config.sidebarOrder.map( + entry => typeof entry === 'string' ? entry : entry.dir, + ); + const versionsByRepo = Object.fromEntries(Object.values(versions).map(v => [v.repo, v])); + const rewriteRaw = createRawLinkRewriter({ + srcDir: resolve(TARGET_DIR), + products: [...configCache.entries()].map(([repoName, config]) => { + const channel = productChannels.get(repoName); + const latest = latestProductVersion({ ...config, latestSource: channel }, versionsByRepo); + return { + contentDir: config.contentDir, + channel, + sections: sections(config), + latestPrefix: `/${config.contentDir}${channel ? `/${latest?.slug ?? channel}` : ''}`, + versionedSections: Object.fromEntries(versions[config.repo].versions.flatMap(version => { + const configPath = join(CONFIG_DIR, `${repoName}.${version.slug}.json`); + return existsSync(configPath) ? [[version.slug, sections(readDocsConfig(configPath))]] : []; + })), + }; + }), + }); + for (const file of collectMarkdownFiles(TARGET_DIR)) { + const original = readFileSync(file, 'utf8'); + const rewritten = await rewriteRaw(original, resolve(file)); + if (rewritten !== original) writeFileSync(file, rewritten); + } + // ------------------------------------------------------------------ // Step 9: Bust Astro cache // ------------------------------------------------------------------ diff --git a/src/build/rawLinkRewrite.spec.ts b/src/build/rawLinkRewrite.spec.ts new file mode 100644 index 00000000..60937c84 --- /dev/null +++ b/src/build/rawLinkRewrite.spec.ts @@ -0,0 +1,41 @@ +import { expect, it } from 'vitest'; +import { parseFrontmatter } from '@astrojs/markdown-remark'; +import { createRawLinkRewriter } from './rawLinkRewrite'; + +const options = { + srcDir: '/docs', + products: [{ contentDir: 'core', sections: ['concepts'], latestPrefix: '/core/v1-16' }], +}; + +it.each(['', '\n', '\uFEFF'])('retains frontmatter with prefix %j while rewriting links', async (prefix) => { + const rewrite = createRawLinkRewriter(options); + const source = '---\ntitle: Guide\n---\n\n[Guide](/concepts/guide/#intro)\n\n[guide]: /concepts/guide/\n\n`[Example](/concepts/example/)`'; + const result = await rewrite(prefix + source, '/docs/core/v1-16/index.md'); + expect(result.startsWith(prefix + '---\ntitle: Guide\n---')).toBe(true); + expect(parseFrontmatter(result).frontmatter.title).toBe('Guide'); + expect(result).toContain('[Guide](/core/v1-16/concepts/guide/#intro)'); + expect(result).toContain('[guide]: /core/v1-16/concepts/guide/'); + expect(result).toContain('`[Example](/concepts/example/)`'); + expect(await rewrite(result, '/docs/core/v1-16/index.md')).toBe(result); +}); + +it('rewrites nested MDX links without rendering LikeC4 or changing fenced examples', async () => { + const rewrite = createRawLinkRewriter(options); + const source = `import LikeC4View from '@components/LikeC4View.astro'; + + + [Guide](/concepts/guide/) + + + + +\`\`\`mdx + +\`\`\` +`; + const result = await rewrite(source, '/docs/core/v1-16/index.mdx'); + expect(result).toContain('[Guide](/core/v1-16/concepts/guide/)'); + expect(result).toContain(''); + expect(result).toContain(''); + expect(result).toContain('```mdx\n\n```'); +}); diff --git a/src/build/rawLinkRewrite.ts b/src/build/rawLinkRewrite.ts new file mode 100644 index 00000000..0247b44a --- /dev/null +++ b/src/build/rawLinkRewrite.ts @@ -0,0 +1,36 @@ +import { createMarkdownProcessor, parseFrontmatter, unified, type RemarkPlugins } from '@astrojs/markdown-remark'; +import { toMarkdown, type Options } from 'mdast-util-to-markdown'; +import { pathToFileURL } from 'node:url'; +import type { Root } from 'mdast'; +import { remarkLinkRewrite, type LinkRewriteOptions } from '../plugins/remark-link-rewrite'; + +/** Apply the site's link rules to raw Markdown/MDX without rendering components. */ +export function createRawLinkRewriter(options: LinkRewriteOptions) { + return async (source: string, filePath: string): Promise => { + const { content } = parseFrontmatter(source, { frontmatter: 'empty-with-spaces' }); + // Astro masks the header in place; the last changed character marks its end. + let headerEnd = source.length; + while (headerEnd > 0 && source[headerEnd - 1] === content[headerEnd - 1]) headerEnd--; + let rewritten = ''; + const remarkPlugins: RemarkPlugins = [ + [remarkLinkRewrite, options], + function serialize(this: { data(key: string): Options['extensions'] }) { + const extensions = this.data('toMarkdownExtensions'); + return (tree: Root) => { + rewritten = toMarkdown(tree, { extensions }); + }; + }, + ]; + if (filePath.endsWith('.mdx')) { + const mdx = await unified({ remarkPlugins, smartypants: false }).createMdxRenderer!( + { syntaxHighlight: false }, { optimize: false }, + ); + await mdx.process(content, filePath, {}); + } else { + const markdown = await createMarkdownProcessor({ syntaxHighlight: false, smartypants: false, remarkPlugins }); + await markdown.render(content, { fileURL: pathToFileURL(filePath) }); + } + // Keep frontmatter unchanged; only the document body is serialized. + return source.slice(0, headerEnd) + '\n\n' + rewritten; + }; +} diff --git a/src/plugins/remark-link-rewrite.ts b/src/plugins/remark-link-rewrite.ts index 7e15ff91..f061155c 100644 --- a/src/plugins/remark-link-rewrite.ts +++ b/src/plugins/remark-link-rewrite.ts @@ -19,7 +19,7 @@ interface ProductConfig { latestPrefix: string; } -interface Options { +export interface LinkRewriteOptions { products: ProductConfig[]; srcDir: string; } @@ -28,7 +28,7 @@ interface Options { * @param options.products - Per-product section lists and optional version overrides * @param options.srcDir - Absolute path to src/content/docs/ */ -export function remarkLinkRewrite(options: Options) { +export function remarkLinkRewrite(options: LinkRewriteOptions) { const { products, srcDir } = options; // Pre-build Sets for all section lookups so we don't re-create them per file. @@ -125,7 +125,7 @@ export function remarkLinkRewrite(options: Options) { : `/${contentDir}`; // Rewrite markdown links: [text](/section/...) → [text](/core/section/...) - visit(tree, 'link', (node) => { + visit(tree, ['link', 'definition'], (node: any) => { node.url = rewriteInternalUrl(node.url, prefix, sections, contentDir); });