From 0ebed5ec15c9644fcbe6b607287ce0affadac70e Mon Sep 17 00:00:00 2001 From: Diip3sh Date: Thu, 8 Oct 2026 00:07:30 +0530 Subject: [PATCH 1/7] Give the CLI one look, and a way to read one finding MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The scan now reads as a single numbered to-do list grouped by area, with a shape per severity (● ◐ ○) so it still reads without colour, the first task marked as where to start, and a status bar at the end with the report, the gate and the next commands. The numbers are what the new `zomb show ` takes: one finding in full, with where, how to fix it, and whether zomb fix does it for you. While it scans, a dot matrix ripples above a checklist of the seven checks, ticking each off as it lands, then collapses to one line. Whole- repo scans end with a calendar of lines added per day and the study's deleted-per-100 numbers drawn as bars; --since leaves both out. zomb fix shows its plan and each step as a checklist, blueprint uses the same headers, and --help gets a block-letter title, the commands in a box and the flags in two columns. Nothing but the terminal text changed: --json, --markdown, the HTML report, the baseline, blueprint.yml, fix's branches and commits, guard, and every exit code are byte-for-byte what they were, checked against the previous commit on the same repos. The Knip warning now prints after the loader, since a line written mid-redraw was drawn over. Co-Authored-By: Claude Opus 5.5 (1M context) --- README.md | 1 + src/cli.ts | 59 +++---- src/fix.ts | 84 +++++---- src/terminal.ts | 413 +++++++++++++++++++++++++++++---------------- test/e2e.test.ts | 7 + test/score.test.ts | 11 +- 6 files changed, 359 insertions(+), 216 deletions(-) diff --git a/README.md b/README.md index 9906e6c..04108c4 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,7 @@ It prints a summary and writes a single HTML report to `.zomb/report.html` (git- ```bash zomb --all # every finding in the terminal, not just the top 5 +zomb show 3 # finding 3 from the list in full: where, why, how to fix zomb --json # an ordered to-do list for an AI agent or CI zomb --since main # only what your branch changed, plus the shortcuts it took zomb --since HEAD # only your uncommitted work diff --git a/src/cli.ts b/src/cli.ts index 435e58c..03935c4 100755 --- a/src/cli.ts +++ b/src/cli.ts @@ -10,7 +10,7 @@ import { parseAll, dependents, touches, clones, orphanRoutes } from './signals.t import { scanRepo, npmAudit, openRoute, hasAuthSignal, securityFindings } from './security.ts'; import { renderReport } from './report.ts'; import { toTasks, fingerprint, toMarkdown } from './tasks.ts'; -import { render, spinner, ui, renderBlueprint } from './terminal.ts'; +import { render, loader, ui, renderBlueprint, renderHelp, renderTask } from './terminal.ts'; import { infer, toRules, breaks, toYaml, readBlueprint, FILE as BLUEPRINT } from './blueprint.ts'; import { fix } from './fix.ts'; import { benchmark } from './benchmark.ts'; @@ -21,7 +21,7 @@ const why = (e: unknown) => (e as { stderr?: string; message?: string }) || {}; import { changesSince, scopeTo, shortcuts } from './diff.ts'; const paint = (tty) => ui({ width: process.stdout.columns || 80, color: Boolean(tty) && !process.env.NO_COLOR }); -const { dim, red, yellow, green, cyan } = paint(process.stderr.isTTY); +const { dim, red, yellow, green } = paint(process.stderr.isTTY); const log = (msg) => process.stderr.write(`${dim('zomb')} ${msg}\n`); const fail = (msg) => (process.stderr.write(`${red('✗')} ${msg}\n`), process.exit(1)); @@ -48,34 +48,14 @@ try { } const { values: opts, positionals } = args; if (opts.help) { - const { bold, dim, cyan, rule } = paint(process.stdout.isTTY); - const cmd = (c, what) => ` ${cyan(c.padEnd(26))}${what}`; - console.log(` -${bold('zomb')} ${dim('a health check for codebases written with AI')} - -${rule('COMMANDS')} -${cmd('zomb [path]', 'scan: security, zombie code, sprawl, architecture')} -${cmd('zomb fix [path]', 'delete dead code on a branch, gated by your build')} -${cmd('zomb blueprint [path]', 'infer the rules this code follows; --write saves them')} -${cmd('zomb guard', 'Claude Code hook: blocks bad edits as they happen')} - -${rule('SCAN')} -${cmd('--all', 'every finding, not just the top 5')} -${cmd('--since ', 'only what changed since a branch or commit')} -${cmd('--json', 'an ordered to-do list for an agent or CI')} -${cmd('--markdown', 'a pull request comment')} -${cmd('--out ', 'where to write the HTML report')} -${cmd('--save-baseline', 'accept today\'s findings; only new ones fail')} -${cmd('--fail-on ', 'exit 1 on new high, medium or low findings')} - -${rule('FIX')} -${cmd('--dry-run', 'show the plan, change nothing')} -${cmd('--no-checks', 'skip the typecheck, lint, test and build gate')} -`); + const { version } = createRequire(import.meta.url)('../package.json'); + console.log(renderHelp({ width: process.stdout.columns || 80, color: Boolean(process.stdout.isTTY) && !process.env.NO_COLOR, version })); process.exit(0); } -// `zomb fix [path]` does the safe clean-up, `zomb blueprint [path]` proposes rules; plain `zomb [path]` reports -const command = ['fix', 'blueprint'].includes(positionals[0]) ? positionals.shift() : 'scan'; +// `zomb fix [path]` does the safe clean-up, `zomb blueprint [path]` proposes rules, `zomb show [path]` explains one finding; plain `zomb [path]` reports +const command = ['fix', 'blueprint', 'show'].includes(positionals[0]) ? positionals.shift() : 'scan'; +const pick = command === 'show' ? Number(positionals.shift()) : 0; +if (command === 'show' && !(Number.isInteger(pick) && pick > 0)) fail(`zomb show takes a finding number from the scan, like ${green('zomb show 1')}`); const LEVELS = { high: ['high'], medium: ['high', 'medium'], low: ['high', 'medium', 'low'] }; const exec = promisify(execFile); const sh = (cmd, args, cwd) => exec(cmd, args, { cwd, maxBuffer: 512 * 1024 * 1024 }).then((r) => r.stdout); @@ -97,9 +77,8 @@ async function knip() { // A config Knip can't load (vitest.config.ts without vitest installed) hides entry points, so its partial output lies. const whyLine = `${why(e).stderr || why(e).message}`.split('\n').find((l) => l.startsWith('ERROR')) || String(why(e).message).split('\n')[0]; const hint = existsSync(path.join(root, 'node_modules')) ? '' : ' Run npm install in the repo first: Knip needs dependencies to read config files.'; - const error = `${whyLine.replace(/^ERROR:\s*/, '')}.${hint}`; - process.stderr.write(`${yellow('!')} Knip failed, so zombie detection is skipped: ${error}\n`); - return { issues: [], error }; + // printed once the loader is gone: a line written while it redraws would be drawn over + return { issues: [], error: `${whyLine.replace(/^ERROR:\s*/, '')}.${hint}` }; } } @@ -146,7 +125,8 @@ const all = [...new Set((await sh('git', ['ls-files', '--cached', '--others', '- const tracked = new Set((await sh('git', ['ls-files'], root)).split('\n').filter(Boolean)); const files = all.filter((f) => CODE.test(f) && !/\.d\.[cm]?ts$/.test(f)); if (!files.length) fail(`no JS/TS files tracked in ${root}`); -const spin = opts.json || opts.markdown ? { step() {}, stop() {} } : spinner(`scanning ${files.length} files`); +const CHECKS: [string, string][] = [['imports', 'what nothing uses'], ['history', 'who wrote what'], ['code', `parse ${files.length} files`], ['copy-paste', 'repeated blocks'], ['routes', 'API routes and auth'], ['secrets', 'keys and .env files'], ['packages', 'npm audit']]; +const spin = opts.json || opts.markdown ? { step() {}, stop() {} } : loader({ title: path.basename(root), label: `scanning ${files.length} files`, steps: CHECKS, color: !process.env.NO_COLOR }); if (!process.stderr.isTTY) log(`scanning ${files.length} JS/TS files in ${root}`); const track = (name, p) => p.then((v) => (spin.step(name), v)); @@ -180,6 +160,7 @@ for (;;) { } const zombieFiles = [...unused, ...onlyTests].filter((f) => !refs.get(f) && !live.has(f)); spin.stop(); +if (knipOut.error) process.stderr.write(`${yellow('!')} Knip failed, so zombie detection is skipped: ${knipOut.error}\n`); const byAgent = await authorship(zombieFiles, agentOf); const zombies = zombieFiles .map((f) => { @@ -207,6 +188,9 @@ const security = securityFindings({ ...scan, dangerous, openRoutes, audit }); // ---------- sprawl ---------- const months = growth(history); const recent = months.filter((m) => Date.now() - Date.parse(`${m.month}-01`) < 92 * 864e5); +// lines added per day (the author's own date), for the activity calendar +const days = new Map(); +for (const c of history) days.set(c.date.slice(0, 10), (days.get(c.date.slice(0, 10)) || 0) + c.files.reduce((s, f) => s + f.added, 0)); const packages = new Map(); for (const [file, p] of parsed) if (!isTest(file)) for (const pkg of p.packages) packages.set(pkg, [...(packages.get(pkg) || []), file]); const dupes = copies && copies.sort((x, y) => y.lines - x.lines); @@ -251,6 +235,14 @@ if (opts.since) { } const tasks = toTasks(data); +// `zomb show `: the same numbers the scan prints +if (command === 'show') { + const t = tasks.find((x) => x.id === pick); + if (!t) fail(tasks.length ? `there ${tasks.length === 1 ? 'is 1 finding' : `are ${tasks.length} findings`}: pick 1 to ${tasks.length}` : 'the scan found nothing to show'); + else console.log(renderTask(t, tasks.length, { repo: data.repo, width: process.stdout.columns || 80, color: Boolean(process.stdout.isTTY) && !process.env.NO_COLOR })); + process.exit(0); +} + if (command === 'blueprint') { const proposal = infer(parsed, { middlewareAuth }); const today = [...parsed].filter(([f]) => !isTest(f)).flatMap(([file, p]) => breaks(toRules(proposal), { file, packages: p.packages, deep: p.deep, route: middlewareAuth ? null : p.route, lines: p.lines }).map((b) => ({ file, ...b }))); @@ -272,7 +264,7 @@ const baselinePath = path.join(root, '.zomb', 'baseline.json'); if (opts['save-baseline']) { await mkdir(path.dirname(baselinePath), { recursive: true }); await writeFile(baselinePath, `${JSON.stringify({ version: 1, commit: data.commit, saved: data.date, fingerprints: [...new Set(tasks.map(fingerprint))].sort() }, null, 2)}\n`); - process.stderr.write(`${green('✓')} saved ${tasks.length} current findings to ${cyan('.zomb/baseline.json')}\n ${dim('From now on only new ones fail --fail-on. Commit this file.')}\n`); + process.stderr.write(`${green('✓')} saved ${tasks.length} current findings to ${green('.zomb/baseline.json')}\n ${dim('From now on only new ones fail --fail-on. Commit this file.')}\n`); process.exit(0); } const baseline = existsSync(baselinePath) ? new Set(JSON.parse(await readFile(baselinePath, 'utf8')).fingerprints) : null; @@ -320,6 +312,7 @@ const lines = render(data, tasks, { audit, knipError: knipOut.error, recent, + days, out: path.relative(process.cwd(), out) || out, failing, failOn: opts['fail-on'], diff --git a/src/fix.ts b/src/fix.ts index 98411b9..9ca7ae6 100644 --- a/src/fix.ts +++ b/src/fix.ts @@ -43,7 +43,12 @@ export const isTooling = (name: string) => const chunk = (list: T[], size: number): T[][] => Array.from({ length: Math.ceil(list.length / size) }, (_, i) => list.slice(i * size, i * size + size)); export async function fix({ root, data, dryRun = false, noChecks = false, color = true, width = 80, log = console.log }: { root: string; data: ScanData; dryRun?: boolean; noChecks?: boolean; color?: boolean; width?: number; log?: (s: string) => void }): Promise { - const { dim, red, yellow, green, cyan, bold, n, plural, row, wrap, rule, frame } = ui({ width, color }); + const { W, dim, red, yellow, green, faint, bold, underline, n, plural, clip, row, head, wrap, dots } = ui({ width, color }); + // a running step shows as ▣ on a terminal, then turns into ☒ with how long it took + const live = Boolean(process.stdout.isTTY) && log === console.log; + let began = 0; + const start = (what: string) => ((began = Date.now()), live && process.stdout.write(` ${bold('▣')} ${what}…`)); + const finish = (what: string, box = dim('☒')) => log(`${live ? '\r\x1b[2K' : ''}${row(`${box} ${what}`, faint(`${((Date.now() - began) / 1000).toFixed(1)}s`), 3)}`); const git = (...args: string[]) => exec('git', args, { cwd: root, maxBuffer: 64 * 1024 * 1024 }).then((r) => r.stdout.trim()); const z = data.zombie; @@ -54,15 +59,13 @@ export async function fix({ root, data, dryRun = false, noChecks = false, color // ── the plan const lines = files.reduce((s, f) => s + f.lines, 0); - const mark = (k) => (k ? cyan('●') : dim('○')); log(''); - for (const l of frame(`${bold('zomb fix')} ${data.repo}`, dryRun ? yellow('dry run') : '', [ - `${mark(files.length)} delete ${plural(files.length, 'dead file')}${lines ? dim(` ${n(lines)} lines`) : ''}`, - `${mark(exportCount)} remove ${plural(exportCount, 'dead export')}`, - `${mark(packages.length)} uninstall ${plural(packages.length, 'unused package')}`, - ])) - log(l); - if (packages.length) for (const l of wrap(dim('packages'), packages)) log(l); + for (const l of head(`${green('⬢ zomb fix')} ${bold(underline(data.repo))}`, dryRun ? yellow('dry run') : dim('the safe clean-up'))) log(l); + const plan: [string, string][] = []; + if (files.length) plan.push([`delete ${plural(files.length, 'dead file')}`, `${n(lines)} lines`]); + if (exportCount) plan.push([`remove ${plural(exportCount, 'dead export')}`, '']); + if (packages.length) plan.push([`uninstall ${plural(packages.length, 'unused package')}`, packages.join(' · ')]); + for (const [what, detail] of plan) log(` ${dim('☐')} ${detail ? `${what.padEnd(30)}${dim(clip(detail, W - 36))}` : what}`); const scripts = z.files.filter((f) => f.script); const loose = z.files.filter((f) => !f.script && !f.committed); const left: [string, string[]][] = [ @@ -70,17 +73,17 @@ export async function fix({ root, data, dryRun = false, noChecks = false, color [`${plural(loose.length, 'dead file')} not committed yet`, loose.map((f) => f.path)], [`${plural(tooling.length, 'tooling package')} configs may load by name`, tooling], ].filter(([, items]) => items.length) as [string, string[]][]; - if (left.length) log(`\n${rule('LEFT FOR YOU', dim('zomb won\'t touch these'))}`); + if (left.length) (log(''), log(dots(bold('left for you'), dim("zomb won't touch these")))); for (const [label, items] of left) { - log(row(`${yellow('!')} ${label}`)); - for (const l of wrap('', items, 3)) log(l); + log(` ${yellow('!')} ${label}`); + for (const l of wrap('', items, 5)) log(l); } - if (!files.length && !exportCount && !packages.length) return log(`\n ${green('✓')} Nothing to clean up.\n`), 0; - if (dryRun) return log(`\n ${dim('Dry run: nothing changed. Run')} ${cyan('zomb fix')} ${dim('to do it.')}\n`), 0; + if (!files.length && !exportCount && !packages.length) return log(`\n ${green('✓')} Nothing to clean up.\n`), 0; + if (dryRun) return log(`\n ${dim('Dry run: nothing changed. Run')} ${green('zomb fix')} ${dim('to do it.')}\n`), 0; // ── safety first: never mix our changes into uncommitted work if (await git('status', '--porcelain', '--untracked-files=no')) { - log(`\n ${red('✗')} You have uncommitted changes. ${dim('Commit or stash them first: zomb fix commits its work on a new branch.')}\n`); + log(`\n ${red('✗')} You have uncommitted changes. ${dim('Commit or stash them first: zomb fix commits its work on a new branch.')}\n`); return 1; } const pkg = existsSync(path.join(root, 'package.json')) ? JSON.parse(readFileSync(path.join(root, 'package.json'), 'utf8')) : {}; @@ -93,16 +96,21 @@ export async function fix({ root, data, dryRun = false, noChecks = false, color () => ({ ok: true }), (e) => ({ ok: false, tail: `${e.stdout || ''}${e.stderr || ''}`.trim().split('\n').slice(-6).join('\n') }), ); - log(`\n${rule('CHECKS', dim('the ones that pass now judge every change'))}`); + log(''); + log(row(`${dim('⬢')} ${bold('Cleaning')} ${dim('one batch at a time, checking each')}`, '', 1)); + // the checks that pass now judge every change const gates: Check[] = []; + const failing: string[] = []; + start('running your checks'); for (const c of checks) { - const r = await runCheck(c); - log(row(`${r.ok ? green('✓') : yellow('✗')} ${c.label}`, r.ok ? '' : dim('already failing, ignored'))); - if (r.ok) gates.push(c); + if ((await runCheck(c)).ok) gates.push(c); + else failing.push(c.label); } + finish(`checks before ${[...gates.map((c) => `${c.label} ${green('✓')}`), ...failing.map((l) => `${l} ${yellow('✗')}`)].join(dim(' · ')) || dim('none found')}`); + if (failing.length) log(` ${dim(`${failing.join(', ')} already failing, so ${failing.length === 1 ? 'it' : 'they'} won't judge the changes`)}`); if (!gates.length && !noChecks) { - log(`\n ${red('✗')} No passing typecheck, lint, test or build script to prove the changes safe.`); - log(` ${dim('Add one to package.json, or run')} ${cyan('zomb fix --no-checks')} ${dim('and review the branch yourself.')}\n`); + log(`\n ${red('✗')} No passing typecheck, lint, test or build script to prove the changes safe.`); + log(` ${dim('Add one to package.json, or run')} ${green('zomb fix --no-checks')} ${dim('and review the branch yourself.')}\n`); return 1; } const verify = async (): Promise<{ ok: boolean; label?: string; tail?: string }> => { @@ -117,7 +125,7 @@ export async function fix({ root, data, dryRun = false, noChecks = false, color let branch = `zomb/fix-${new Date().toLocaleDateString('en-CA')}`; for (let i = 2; await git('rev-parse', '--verify', '--quiet', branch).then(() => true, () => false); i++) branch = `zomb/fix-${new Date().toLocaleDateString('en-CA')}-${i}`; await git('switch', '-q', '-c', branch); - log(`\n${rule('CLEANING', cyan(branch))}`); + log(row(`${dim('☒')} new branch ${branch}`, '', 3)); const undo = async (reinstall?: boolean) => { await git('reset', '-q', '--hard', 'HEAD'); // only our uncommitted batch: the tree was clean when we started @@ -154,53 +162,59 @@ export async function fix({ root, data, dryRun = false, noChecks = false, color // ── 1. dead files for (const group of chunk(files, 10)) { + start(`deleting ${plural(group.length, 'dead file')}`); const ok = await batch(group, (list) => git('rm', '-q', '--', ...list.map((f) => f.path)), { one: (f) => f.path }); if (ok.length) { const lines = ok.reduce((s, f) => s + f.lines, 0); await commit(`zomb fix: delete ${plural(ok.length, 'unused file')} (${n(lines)} lines)`); done.files.push(...ok); done.lines += lines; - log(row(`${green('✓')} deleted ${plural(ok.length, 'file')}`, dim(`${n(lines)} lines`))); - } + finish(`deleted ${plural(ok.length, 'file')} ${dim(`−${n(lines)} lines`)}`); + } else finish(dim('deleted no files: the checks need them'), yellow('☒')); } // ── 2. dead exports: Knip's own fixer rewrites the export keywords (fresh, so exports freed by step 1 go too) if (exportCount || done.files.length) { + start('removing dead exports'); const knip = path.join(path.dirname(createRequire(import.meta.url).resolve('knip')), '..', 'bin', 'knip.js'); const ok = await batch(['exports'], () => exec(process.execPath, [knip, '--fix', '--fix-type', 'exports,types', '--no-progress', '--no-exit-code'], { cwd: root, maxBuffer: 64 * 1024 * 1024 }).catch(() => {}), { one: () => 'dead exports' }); if (ok.length && (await git('status', '--porcelain', '--untracked-files=no'))) { await git('add', '-A'); await commit('zomb fix: remove unused exports'); done.exports = true; - log(row(`${green('✓')} removed dead exports`)); - } + finish('removed dead exports'); + } else finish(dim('no dead exports to remove')); } // ── 3. unused packages if (packages.length) { const remove = { npm: ['uninstall'], pnpm: ['remove'], yarn: ['remove'], bun: ['remove'] }[pm]; + start(`uninstalling ${plural(packages.length, 'package')}`); const ok = await batch(packages, (list) => exec(pm, [...remove, ...list], { cwd: root, maxBuffer: 64 * 1024 * 1024 }), { reinstall: true }); if (ok.length) { await git('add', '-A'); await commit(`zomb fix: uninstall ${plural(ok.length, 'unused package')}\n\n${ok.join(', ')}`); done.packages = ok; - for (const l of wrap(`${green('✓')} uninstalled`, ok)) log(l); - } + finish(`uninstalled ${plural(ok.length, 'package')} ${dim(clip(ok.join(' · '), W - 40))}`); + } else finish(dim('uninstalled nothing: the checks need them'), yellow('☒')); } // ── summary - log(`\n${rule('DONE', green(`−${n(done.lines)} lines`))}`); - log(row(`${plural(done.files.length, 'file')} deleted${dim(' · ')}${done.exports ? `dead exports removed${dim(' · ')}` : ''}${plural(done.packages.length, 'package')} uninstalled`)); - for (const k of kept) log(row(`${yellow('kept')} ${k.item}`, dim(`${k.why}, so it isn't dead`))); + log(''); + const kepts = kept.map((k) => row(`${yellow('kept')} ${k.item}`, dim(`${k.why}, so it isn't dead`), 3)); if (!done.files.length && !done.exports && !done.packages.length) { await git('switch', '-q', original); await git('branch', '-q', '-D', branch); - log(` ${dim('Nothing could be removed safely; deleted the empty branch.')}\n`); + kepts.forEach((l) => log(l)); + log(` ${yellow('!')} ${dim('Nothing could be removed safely; deleted the empty branch.')}\n`); return 0; } + log(row(`${green('✓')} ${bold('Cleaned')} ${dim('on')} ${branch}`, green(`−${n(done.lines)} lines`), 1)); + log(` ${[plural(done.files.length, 'file'), ...(done.exports ? ['dead exports'] : []), plural(done.packages.length, 'package'), `${n(kept.length)} kept`].join(dim(' · '))}`); + kepts.forEach((l) => log(l)); log(''); - log(` ${dim('review')} ${cyan(`git diff ${original}...${branch}`)}`); - log(` ${dim('keep it')} ${cyan(`git switch ${original} && git merge ${branch}`)}`); - log(` ${dim('drop it')} ${cyan(`git switch ${original} && git branch -D ${branch}`)}\n`); + log(` ${dim('review')} ${green(`git diff ${original}...${branch}`)}`); + log(` ${dim('keep ')} ${green(`git switch ${original} && git merge ${branch}`)}`); + log(` ${dim('drop ')} ${green(`git switch ${original} && git branch -D ${branch}`)}\n`); return 0; } diff --git a/src/terminal.ts b/src/terminal.ts index 1902a41..b19c5f4 100644 --- a/src/terminal.ts +++ b/src/terminal.ts @@ -1,28 +1,34 @@ -// The terminal summary. Fits the window: nothing wraps, long paths shorten from the left, numbers line up on the right. +// The terminal look for every zomb command. Fits the window: nothing wraps, long paths shorten from the left, numbers line up on the right. +// One visual language: ⬢ marks zomb at work, ● ◐ ○ are high, medium and low, ☒ ▣ ☐ are done, running and next. import type { Month, Rule, ScanData, Severity, Task } from './types.ts'; const ANSI = /\x1b\[[0-9;]*m/g; export const visible = (s: unknown) => String(s).replace(ANSI, '').length; +const DAY = 864e5; -// One look for every zomb command: colors, badges, section rules, a header frame, and rows that fit the window. export function ui({ width = 80, color = true, all = false }: { width?: number; color?: boolean; all?: boolean } = {}) { // capped at 80: on a wide window, numbers pushed far right are hard to match to their file const W = Math.max(48, Math.min(width - 1, 80)); const paint = (code: number | string) => (s: unknown) => (color ? `\x1b[${code}m${s}\x1b[0m` : String(s)); - const [bold, dim, red, yellow, green, cyan] = [paint(1), paint(2), paint(31), paint(33), paint(32), paint(36)]; - const badge = { high: paint('1;41;97'), medium: paint('1;43;30'), low: paint('2;7') }; + const [bold, dim, red, yellow, green, faint, underline] = [paint(1), paint(2), paint(31), paint(33), paint(32), paint(90), paint(4)]; + // the shape carries the severity too, so it still reads without colour + const mark: Record = { high: red('●'), medium: yellow('◐'), low: dim('○') }; const n = (x: number) => x.toLocaleString('en-US'); const plural = (k: number, word: string) => `${n(k)} ${word}${k === 1 ? '' : 's'}`; // a path that must fit in `w` columns loses its start, never its file name const fit = (s: string, w: number) => (visible(s) <= w ? s : `…${s.slice(-(w - 1))}`); + // prose that must fit loses its end + const clip = (s: string, w: number) => (s.length <= w ? s : `${s.slice(0, Math.max(w - 1, 1))}…`); // left text + right text on one line, left shortened to make room const row = (left: string, right = '', indent = 2) => { const room = W - indent - visible(right) - (right ? 2 : 0); const l = visible(left) > room ? fit(left.replace(ANSI, ''), room) : left; return right ? `${' '.repeat(indent)}${l}${' '.repeat(Math.max(room - visible(l), 0) + 2)}${right}` : `${' '.repeat(indent)}${l}`; }; + // a title line: the meta goes on its own line when both don't fit + const head = (left: string, right = '') => (visible(left) + visible(right) + 3 <= W ? [row(left, right, 1)] : [row(left, '', 1), ...(right ? [row(right, '', 1)] : [])]); // words that wrap onto indented lines instead of running off the edge - const wrap = (label: string, words: string[], indent = 2) => { + const wrap = (label: string, words: string[], indent = 2, sep = dim(' · ')) => { const pad = ' '.repeat(indent + visible(label) + 1); const lines: string[] = []; let cur = `${' '.repeat(indent)}${label} `; @@ -30,203 +36,320 @@ export function ui({ width = 80, color = true, all = false }: { width?: number; if (visible(cur) + visible(w) + 3 > W && visible(cur) > pad.length) (lines.push(cur.trimEnd()), (cur = pad)); const room = W - visible(cur) - 3; if (visible(w) > room) w = fit(w.replace(ANSI, ''), Math.max(room, 8)); // one item wider than the line - cur += cur === pad || cur.endsWith(`${label} `) ? w : `${dim(' · ')}${w}`; + cur += cur === pad || cur.endsWith(`${label} `) ? w : `${sep}${w}`; } return [...lines, cur]; }; - const rule = (title: string, right = '') => `${bold(title)} ${dim('─'.repeat(Math.max(W - visible(title) - visible(right) - (right ? 2 : 1), 1)))}${right ? ` ${right}` : ''}`; - const more = (list: unknown[], k = 5) => (!all && list.length > k ? [dim(` …and ${n(list.length - k)} more (zomb --all)`)] : []); - // the rounded header box: a title line with something on the right, then rows under a divider - const frame = (title: string, meta = '', rows: string[] = []) => { - const line = (s: string) => `${dim('│')} ${s}${' '.repeat(Math.max(W - 4 - visible(s), 0))} ${dim('│')}`; - const top = visible(title) + visible(meta) + 2 <= W - 4 ? `${title}${' '.repeat(W - 4 - visible(title) - visible(meta))}${meta}` : title; - return [dim(`╭${'─'.repeat(W - 2)}╮`), line(top), ...(rows.length ? [dim(`├${'─'.repeat(W - 2)}┤`), ...rows.map(line)] : []), dim(`╰${'─'.repeat(W - 2)}╯`)]; + // a section heading: its name, a dotted leader, and a summary on the right + const dots = (left: string, right = '') => ` ${left} ${faint('·'.repeat(Math.max(W - visible(left) - visible(right) - (right ? 3 : 2), 3)))}${right ? ` ${right}` : ''}`; + // the status bar: values over their labels, as many columns as fit + const bar = (cells: [string, string][]) => { + const top: string[] = []; + const bottom: string[] = []; + let used = 1; + for (const [value, label] of cells) { + const w = Math.max(visible(value), visible(label)); + if (used + w > W) break; + top.push(value + ' '.repeat(w - visible(value))); + bottom.push(label + ' '.repeat(w - visible(label))); + used += w + 3; + } + const line = (xs: string[]) => ` ${xs.join(' ')}`.replace(/ +$/, ''); + return [` ${faint('─'.repeat(W - 1))}`, line(top), line(bottom)]; }; - return { W, paint, bold, dim, red, yellow, green, cyan, badge, n, plural, fit, row, wrap, rule, more, frame }; + return { W, all, bold, dim, red, yellow, green, faint, underline, mark, n, plural, fit, clip, row, head, wrap, dots, bar }; } +const AREAS: [Task['area'], string][] = [['security', 'security'], ['shortcuts', 'shortcuts'], ['blueprint', 'blueprint'], ['zombie', 'zombie code'], ['sprawl', 'sprawl'], ['architecture', 'architecture']]; +const short = (ref: string) => (/^[0-9a-f]{40}$/.test(ref) ? ref.slice(0, 7) : ref); + export function render( data: ScanData, tasks: Task[], - { width = 80, color = true, all = false, audit = {}, knipError, recent = [], out, failing = [], failOn, baseline }: { width?: number; color?: boolean; all?: boolean; audit?: ScanData['security']['audit']; knipError?: string; recent?: Month[]; out?: string; failing?: unknown[]; failOn?: string; baseline?: unknown } = {}, + { width = 80, color = true, all = false, audit = {}, knipError, recent = [], days, out, failing = [], failOn, baseline }: { width?: number; color?: boolean; all?: boolean; audit?: ScanData['security']['audit']; knipError?: string; recent?: Month[]; days?: Map; out?: string; failing?: unknown[]; failOn?: string; baseline?: unknown } = {}, ): string[] { - const { W, bold, dim, red, yellow, green, cyan, badge, n, plural, fit, row, wrap, rule, more, frame } = ui({ width, color, all }); - const cap = (list: T[], k = 5) => (all ? list : list.slice(0, k)); - - const L: string[] = []; - const sec = data.security.findings; - const high = sec.filter((f) => f.severity === 'high').length; + const u = ui({ width, color, all }); + const { W, bold, dim, red, yellow, green, faint, underline, mark, n, plural, fit, clip, head, wrap, dots, bar } = u; const z = data.zombie; - const zLines = z.files.reduce((s, f) => s + f.lines, 0); - const sp = data.sprawl; const a = data.architecture; - const added = recent.reduce((s, m) => s + m.added, 0), deleted = recent.reduce((s, m) => s + m.deleted, 0); + const zLines = z.files.reduce((s, f) => s + f.lines, 0); + const of = (area: Task['area']) => tasks.filter((t) => t.area === area); + const count = (list: Task[], sev: Severity) => list.filter((t) => t.severity === sev).length; + const L: string[] = []; - // ── header: what was scanned, and one line per area - const dot = (tone) => ({ bad: red('●'), warn: yellow('●'), ok: green('●') })[tone]; - const summary = [ - ['security', sec.length ? `${high ? `${n(high)} high` : ''}${high && sec.length - high ? ' · ' : ''}${sec.length - high ? `${n(sec.length - high)} medium` : ''}` : 'nothing found', high ? 'bad' : sec.length ? 'warn' : 'ok'], - ...(data.blueprint ? [['blueprint', data.blueprint.broken.length ? `${plural(data.blueprint.broken.length, 'rule break')}` : 'followed', data.blueprint.broken.length ? 'bad' : 'ok']] : []), - ...(data.shortcuts ? [['shortcuts', data.shortcuts.length ? plural(data.shortcuts.length, 'place') : 'none', data.shortcuts.some((c) => c.severity !== 'low') ? 'warn' : 'ok']] : []), - ['zombie code', knipError ? 'skipped' : zLines ? `${n(zLines)} lines · ${plural(z.files.length, 'file')}` : 'none', knipError ? 'warn' : zLines ? 'warn' : 'ok'], - ['sprawl', [sp.overlaps.length && plural(sp.overlaps.length, 'library overlap'), sp.dupes?.length && plural(sp.dupes.length, 'copy-paste'), sp.versions.length && plural(sp.versions.length, 'versioned copy').replace('copys', 'copies')].filter(Boolean).join(' · ') || 'none', sp.overlaps.length || sp.dupes?.length || sp.versions.length ? 'warn' : 'ok'], - ['architecture', [plural(a.cycles.length, 'cycle'), a.big.length && `${n(a.big.length)} over 500 lines`].filter(Boolean).join(' · '), a.cycles.length ? 'warn' : a.big.length ? 'warn' : 'ok'], + // ── header: the repo, its size, and one status mark per area + L.push(...head(`${green('⬢ zomb')} ${bold(underline(clip(data.repo, W - 24)))}${dim(` [${n(tasks.length)}]`)}`, dim(`${n(data.files)} files · ${n(data.lines)} lines${data.since ? ` · since ${short(data.since)}` : ''}`))); + const tone = { bad: red('●'), warn: yellow('◐'), ok: green('✓') }; + const status: [string, keyof typeof tone][] = [ + ['security', count(of('security'), 'high') ? 'bad' : of('security').length ? 'warn' : 'ok'], + ...(data.blueprint ? [['blueprint', data.blueprint.broken.length ? 'bad' : 'ok'] as [string, keyof typeof tone]] : []), + ...(data.shortcuts ? [['shortcuts', data.shortcuts.length ? 'warn' : 'ok'] as [string, keyof typeof tone]] : []), + ['zombie code', knipError || of('zombie').length ? 'warn' : 'ok'], + ['sprawl', of('sprawl').length ? 'warn' : 'ok'], + ['architecture', of('architecture').length ? 'warn' : 'ok'], ]; - const title = `${bold('zomb')} ${data.repo}`; - const meta = dim(`${n(data.files)} files · ${n(data.lines)} lines${data.since ? ` · since ${/^[0-9a-f]{40}$/.test(data.since) ? data.since.slice(0, 7) : data.since}` : ''}`); - L.push(...frame(title, meta, summary.map(([label, value, tone]) => `${dot(tone)} ${label.padEnd(14)}${fit(value, W - 21)}`))); + L.push(...wrap('', status.map(([label, t]) => `${tone[t]} ${label}`), 0, ' ')); - // ── security - if (sec.length || audit.skipped) { - L.push('', rule('SECURITY', high ? red(`${n(high)} high`) : sec.length ? yellow(`${n(sec.length)} medium`) : green('nothing found'))); - for (const f of cap(sec)) { - L.push(row(`${badge[f.severity](f.severity === 'high' ? ' HIGH ' : ' MED ')} ${f.title}`, '', 1)); - L.push(row(dim(f.where), '', 9)); + // ── one numbered to-do list, grouped by area. The numbers are what `zomb show ` takes. + const severities = (list: Task[]) => [count(list, 'high') && red(`${n(count(list, 'high'))} high`), count(list, 'medium') && yellow(`${n(count(list, 'medium'))} medium`), count(list, 'low') && dim(`${n(count(list, 'low'))} low`)].filter(Boolean).join(dim(' · ')); + const summary: Record string> = { + security: severities, + shortcuts: severities, + blueprint: (list) => red(plural(list.length, 'break')), + zombie: (list) => (zLines ? `${bold(n(zLines))} ${dim(`lines · ${plural(z.files.length, 'file')}`)}` : dim(plural(list.length, 'task'))), + sprawl: (list) => dim(`${plural(list.length, 'thing')} to tidy`), + architecture: (list) => (a.cycles.length ? yellow(plural(a.cycles.length, 'cycle')) : dim(plural(list.length, 'task'))), + }; + const added = recent.reduce((s, m) => s + m.added, 0); + const deleted = recent.reduce((s, m) => s + m.deleted, 0); + const byAgent = z.files.filter((f) => f.share >= 0.5).length; + const notes: Record = { + security: audit.skipped ? [` ${dim(clip(`packages not checked: ${audit.skipped}`, W - 3))}`] : [], + shortcuts: [], + blueprint: [], + zombie: byAgent ? [` ${dim(`${byAgent} of ${plural(z.files.length, 'dead file')} written by an AI agent`)}`] : [], + sprawl: added && !data.since ? [` ${dim(clip(`+${n(added)} −${n(deleted)} lines in 3 months${data.benchmark ? '' : ` · ${Math.round((deleted / added) * 100)} deleted per 100 added`}`, W - 3))}`] : [], + architecture: [...(a.shared.length > 1 ? wrap(dim('shared code in'), a.shared, 3) : []), ...(a.deep.length ? [` ${dim(`${plural(a.deep.length, 'file')} with ../../../ imports`)}`] : [])], + }; + const first = tasks[0]; + const item = (t: Task) => { + const lead = ` ${faint(`${String(t.id).padStart(3)}.`)} ${mark[t.severity]} `; + const tags = [t.how.startsWith('Only its own tests') && dim('tests only'), t.safe && green('fix'), t.new && yellow('new')].filter(Boolean).join(' '); + const where = t.where && !t.title.includes(t.where) ? t.where : ''; + // the first task is where to start + const hot = t === first; + const text = clip(t.title, W - visible(lead) - (tags ? visible(tags) + 2 : 0) - (hot ? 4 : 0)); + const left = `${lead}${hot ? `${yellow(underline(text))}${yellow(' (!)')}` : text}`; + const right = [where && dim(where), tags].filter(Boolean).join(' '); + const line = (r: string) => (r ? `${left}${' '.repeat(Math.max(W - visible(left) - visible(r), 2))}${r}` : left); + if (visible(left) + visible(right) + 2 <= W) return [line(right)]; + return [line(tags), ...(where ? [`${' '.repeat(visible(lead))}${dim(fit(where, W - visible(lead)))}`] : [])]; + }; + for (const [area, name] of AREAS) { + const list = of(area); + if (area === 'zombie' && knipError) { + L.push('', dots(bold(name), yellow('skipped')), ` ${dim(clip(knipError, W - 3))}`); + continue; } - L.push(...more(sec)); - if (audit.skipped) L.push(row(dim(`packages not checked: ${audit.skipped}`))); + if (!list.length && !notes[area].length) continue; + L.push('', dots(bold(name), list.length ? summary[area](list) : '')); + for (const t of all ? list : list.slice(0, 5)) L.push(...item(t)); + if (!all && list.length > 5) L.push(`${' '.repeat(9)}${faint(`+ ${n(list.length - 5)} more`)} ${dim('zomb --all')}`); + L.push(...notes[area]); } + if (!tasks.length) L.push('', ` ${green('✓')} ${bold('Nothing to fix.')}`); - // ── shortcuts (diff mode) - if (data.shortcuts?.length) { - L.push('', rule('SHORTCUTS', yellow(`${n(data.shortcuts.length)} in this change`))); - for (const c of cap(data.shortcuts)) { - L.push(row(`${badge[c.severity](c.severity === 'high' ? ' HIGH ' : c.severity === 'medium' ? ' MED ' : ' LOW ')} ${c.kind}`, '', 1)); - L.push(row(dim(c.line ? `${c.file}:${c.line}` : c.file), '', 9)); - } - L.push(...more(data.shortcuts)); - } + // ── whole-repo scans only: when the code was written, and how it compares with the study + if (!data.since) L.push(...activity(u, days, data.benchmark)); - // ── the approved rules this code breaks - const bp = data.blueprint?.broken || []; - if (bp.length) { - L.push('', rule('BLUEPRINT', red(plural(bp.length, 'break')))); - for (const b of cap(bp)) L.push(row(`${red('✗')} ${b.why}`), row(dim(b.file), '', 4)); - L.push(...more(bp)); + // ── the tally, then the status bar + const safe = tasks.filter((t) => t.safe).length; + const fresh = baseline ? tasks.filter((t) => t.new).length : null; + if (tasks.length) { + L.push(''); + // who does what: zomb fix the mechanical part, /zomb-clean in Claude Code the rest + const who = [...(safe ? [dim(`${Math.round((safe / tasks.length) * 100)}% can be fixed by`), `${green('zomb fix')}${dim(tasks.length > safe ? ',' : '.')}`] : []), ...(tasks.length > safe ? [dim(safe ? 'the rest with' : 'Work through them with'), green('/zomb-clean'), dim('in Claude Code.')] : [])]; + L.push(...wrap('', who, 0, ' ')); + const tally: [number, (s: unknown) => string, string][] = [[count(tasks, 'high'), red, 'high'], [count(tasks, 'medium'), yellow, 'medium'], [count(tasks, 'low'), String, 'low'], [safe, green, 'fixable'], [fresh ?? 0, yellow, 'new since the baseline']]; + // a zero is noise, except the baseline count: "0 new" is the news + L.push(...wrap('', tally.filter(([k, , label]) => k || (label.startsWith('new') && fresh !== null)).map(([k, paint, label]) => `${paint(n(k))} ${dim(label)}`), 0)); } + // most useful first: a narrow window drops cells from the right + const cells: [string, string][] = []; + if (failOn) cells.push([failing.length ? red(`✗ ${plural(failing.length, 'finding')}`) : green('✓ pass'), dim(`fail-on ${failOn}${baseline ? ', new only' : ''}`)]); + if (out) cells.push([fit(out, 28), dim('report')]); + if (safe) cells.push([green('zomb fix'), dim(`${n(safe)} safe`)]); + if (tasks.length - safe) cells.push([green('/zomb-clean'), dim(`${n(tasks.length - safe)} to review`)]); + if (tasks.length) cells.push([green('zomb show 1'), dim('the first in full')]); + if (cells.length) L.push('', ...bar(cells)); + return L; +} - // ── zombie code, grouped by folder - if (knipError) L.push('', rule('ZOMBIE CODE', yellow('skipped')), row(dim(fit(knipError, W - 2)))); - else if (z.files.length || z.packages.length || z.maybe.length) { - L.push('', rule('ZOMBIE CODE', `${bold(n(zLines))} ${dim('lines')}`)); - const groups = new Map(); - for (const f of z.files) { - const dir = f.path.includes('/') ? f.path.slice(0, f.path.lastIndexOf('/') + 1) : './'; - groups.set(dir, [...(groups.get(dir) || []), f]); +// A calendar of lines added per day, then this repo's deletion habit against the study, as bars. +function activity({ W, bold, dim, green, yellow, faint, n, plural, row, dots }: ReturnType, days: Map | undefined, bm: ScanData['benchmark']): string[] { + const L: string[] = []; + const weeks = Math.min(26, Math.floor((W - 6) / 2)); + const end = Date.parse(new Date().toLocaleDateString('en-CA')); + const start = end - new Date(end).getUTCDay() * DAY - (weeks - 1) * 7 * DAY; + const seen = [...(days || [])].filter(([d]) => Date.parse(d) >= start && Date.parse(d) <= end); + if (seen.length) { + const max = Math.max(...seen.map(([, v]) => v)); + const cell = (v = 0) => (v ? green('░▒▓█'[Math.min(3, Math.floor((v / max) * 4))].repeat(2)) : faint(' ·')); + const months = Array(5 + weeks * 2).fill(' '); + let free = 0; + for (let c = 0, prev = -1; c < weeks; c++) { + const m = new Date(start + c * 7 * DAY).getUTCMonth(); + const at = 6 + c * 2; + if (m !== prev && at >= free) { + [...'JanFebMarAprMayJunJulAugSepOctNovDec'.slice(m * 3, m * 3 + 3)].forEach((ch, k) => (months[at + k] = ch)); + free = at + 4; + } + prev = m; } - const sorted = [...groups].sort((x, y) => y[1].reduce((s, f) => s + f.lines, 0) - x[1].reduce((s, f) => s + f.lines, 0)); - for (const [dir, list] of cap(sorted)) { - const total = list.reduce((s, f) => s + f.lines, 0); - L.push(row(dim(dir), dim(`${plural(list.length, 'file')} ${n(total).padStart(6)}`))); - for (const f of cap(list)) L.push(row(`${f.path.slice(dir === './' ? 0 : dir.length)}${f.script ? yellow(' run by hand?') : f.why.startsWith('Only') ? dim(' tests only') : ''}`, n(f.lines).padStart(6), 4)); - if (!all && list.length > 5) L.push(dim(` …and ${n(list.length - 5)} more`)); + L.push('', dots(bold('activity'), dim('lines added per day')), dim(months.join('').trimEnd())); + for (let r = 0; r < 7; r++) { + const cells: string[] = []; + for (let c = 0; c < weeks; c++) { + const d = start + (c * 7 + r) * DAY; + if (d <= end) cells.push(cell(days?.get(new Date(d).toISOString().slice(0, 10)))); + } + L.push(`${dim(['', ' Mon', '', ' Wed', '', ' Fri', ''][r].padEnd(5))}${cells.join('')}`.trimEnd()); } - L.push(...more(sorted)); - const byAgent = z.files.filter((f) => f.share >= 0.5).length; - if (z.files.length) L.push(row(dim(`nothing imports or names these${byAgent ? ` · ${byAgent} of ${z.files.length} written by an AI agent` : ''}`))); - if (z.packages.length) L.push('', ...wrap(dim('unused packages'), z.packages.map((p) => p.name))); - if (z.maybe.length) L.push('', ...wrap(dim('maybe unused routes'), z.maybe.map((m) => m.url))); - const dead = z.exports.reduce((s, e) => s + e.names.length, 0); - if (dead) L.push(row(dim(`+ ${plural(dead, 'export')} nothing imports, in ${plural(z.exports.length, 'file')}`))); - } - - // ── sprawl - const sprawlRows: string[] = []; - for (const o of sp.overlaps) sprawlRows.push(...wrap(dim(o.job.padEnd(14)), o.libraries.map((l) => `${l.name} ${dim(n(l.files))}`))); - for (const v of cap(sp.versions)) sprawlRows.push(row(v.path, dim(v.original ? 'has original' : v.files ? plural(v.files, 'file') : 'copy'))); - for (const x of cap(sp.names)) sprawlRows.push(row(x.name, dim(`defined ${x.files.length}×`))); - if (sp.dupes?.length) sprawlRows.push(row(`${plural(sp.dupes.length, 'copy-pasted block')}`, dim(`${n(sp.dupes.reduce((s, d) => s + d.lines, 0))} lines`))); - const growthLine = data.since ? '' : added ? `${dim('+')}${n(added)} ${dim('−')}${n(deleted)} ${dim('3 mo')}` : dim('no commits in 3 months'); - if (sprawlRows.length || growthLine) { - L.push('', rule('SPRAWL', growthLine)); - if (added && !data.since) L.push(row(dim(`${Math.round((deleted / added) * 100)} lines deleted for every 100 added`))); - L.push(...sprawlRows); - } - - // ── architecture - if (a.cycles.length || a.big.length || a.deep.length || a.shared.length > 1) { - L.push('', rule('ARCHITECTURE', a.cycles.length ? yellow(plural(a.cycles.length, 'cycle')) : green('no cycles'))); - for (const c of cap(a.cycles, 3)) L.push(row(`${dim('cycle')} ${c.slice(0, 3).map((f) => f.split('/').pop()).join(dim(' → '))}${c.length > 3 ? dim(` +${c.length - 3}`) : ''}`)); - for (const b of cap(a.big)) L.push(row(b.file, `${n(b.lines).padStart(6)}`)); - if (a.shared.length > 1) L.push(...wrap(dim('shared code in'), a.shared)); - if (a.deep.length) L.push(row(dim(`${plural(a.deep.length, 'file')} with ../../../ imports`))); + const total = seen.reduce((s, [, v]) => s + v, 0); + L.push(row(dim(`${n(total)} lines on ${plural(seen.length, 'day')}${bm ? ` · agents wrote ${bm.agentShare}%` : ''}`), `${dim('less')} ${faint('·')} ${green('░ ▒ ▓ █')} ${dim('more')}`, 1)); } - - // ── how this repo compares with the study - const bm = data.benchmark; if (bm) { + const rows = [{ name: 'this repo', v: bm.deletedPer100, me: true }, { name: 'humans', v: bm.humans, me: false }, ...bm.agents.map((x) => ({ name: x.name, v: x.deletedPer100, me: false }))].sort((x, y) => y.v - x.v); + const room = W - 40; + const scale = room / Math.max(...rows.map((x) => x.v), 1); const worse = bm.deletedPer100 < bm.humans; - L.push('', rule('VS 336 PUBLIC REPOS', dim(`agents wrote ${bm.agentShare}% of your new code`))); - L.push(row(`this repo deletes ${bold(String(bm.deletedPer100))} lines per 100 added`, (worse ? yellow : green)(worse ? 'below the human average' : 'above the human average'))); - L.push(...wrap(dim('study'), [`humans ${bm.humans}`, ...bm.agents.map((a) => `${a.name} ${a.deletedPer100}`)])); - if (bm.medianCommit > bm.humanMedianCommit * 1.5) L.push(row(dim(`your median commit adds ${n(bm.medianCommit)} lines; a human commit in the study adds ${n(bm.humanMedianCommit)}`))); + L.push('', dots(bold('deleted per 100 added'), dim(`vs ${n(bm.study.repos)} public repos`))); + for (const x of rows) { + const len = x.v * scale; + const drawn = `${'█'.repeat(Math.floor(len))}${len % 1 >= 0.5 ? '▌' : ''}`; + const human = x.name === 'humans'; + const label = x.name.padEnd(11); + L.push(` ${x.me ? bold(label) : human ? label : dim(label)}${(x.me ? green : human ? String : faint)(drawn)}${' '.repeat(room - drawn.length + 2)}${(x.me ? bold : dim)(String(x.v).padStart(3))}${x.me ? ` ${(worse ? yellow : green)(worse ? 'below humans' : 'above humans')}` : ''}`); + } + if (bm.medianCommit > bm.humanMedianCommit * 1.5) L.push(` ${dim(`your median commit adds ${n(bm.medianCommit)} lines; a human one in the study adds ${n(bm.humanMedianCommit)}`)}`); } - - // ── what to do next - const safe = tasks.filter((t) => t.safe).length; - const fresh = baseline ? tasks.filter((t) => t.new).length : null; - L.push('', rule('NEXT', dim(`${plural(tasks.length, 'task')}`))); - if (safe) L.push(row(`${cyan('zomb fix')} ${dim('deletes the dead code on a branch, checking your build')}`, dim(`${n(safe)} safe`))); - if (tasks.length - safe) L.push(row(`${cyan('/zomb-clean')} ${dim('in Claude Code for the rest')}`, dim(`${n(tasks.length - safe)} need a look`))); - if (fresh !== null) L.push(row(dim(`${plural(fresh, 'finding')} new since the baseline`))); - if (out) L.push(row(`${dim('report')} ${fit(out, W - 10)}`)); - if (failOn) L.push(row(failing.length ? red(`✗ ${plural(failing.length, 'finding')} at or above ${failOn}${baseline ? ', new since the baseline' : ''}`) : green(`✓ nothing ${baseline ? 'new ' : ''}at or above ${failOn}`))); return L; } +// `zomb show `: one finding in full, with what to do about it. +export function renderTask(t: Task, total: number, { repo, width = 80, color = true }: { repo: string; width?: number; color?: boolean }): string { + const { W, bold, dim, green, mark, n, clip, head, wrap } = ui({ width, color }); + const L = ['', ...head(`${green('⬢ zomb')} ${bold(clip(repo, W - 30))}`, dim(`finding ${n(t.id)} of ${n(total)}`)), '']; + L.push(` ${mark[t.severity]} ${bold(clip(t.title, W - 6))}`, ` ${dim(`${t.area} · ${t.severity}${t.new ? ' · new since the baseline' : ''}`)}`, ''); + if (t.where) L.push(...wrap(dim('where'), t.where.split(/,\s+/), 3, dim(', '))); + L.push(...wrap(dim('fix '), t.how.split(' '), 3, ' ')); + L.push(t.safe ? ` ${dim('who ')} ${green('zomb fix')} ${dim('does this for you, checking your build')}` : ` ${dim('who ')} ${dim('you, or')} ${green('/zomb-clean')} ${dim('in Claude Code')}`); + if (t.id < total) L.push('', ` ${dim('next ')} ${green(`zomb show ${t.id + 1}`)}`); + return `${L.join('\n')}\n`; +} + +// `zomb --help`: the block-letter title, the commands in a box, the flags in columns. +export function renderHelp({ width = 80, color = true, version = '' }: { width?: number; color?: boolean; version?: string } = {}): string { + const { W, bold, dim, green, faint, clip, row } = ui({ width, color }); + const FONT: Record = { Z: ['#####', ' # ', ' # ', ' # ', '#####'], O: [' ### ', '# #', '# #', '# #', ' ### '], M: ['# #', '## ##', '# # #', '# #', '# #'], B: ['#### ', '# #', '#### ', '# #', '#### '] }; + // each pixel row is a line of ▀: the lower half stays empty, which gives the letters their stripes + const banner = [0, 1, 2, 3, 4].map((r) => green(` ${[...'ZOMB'].map((ch) => FONT[ch][r].replace(/#/g, '▀▀').replace(/ /g, ' ')).join(' ')}`.trimEnd())); + const commands = [ + ['zomb [path]', 'scan: security, zombie code, sprawl, architecture'], + ['zomb show [path]', 'one finding in full: where, why, how to fix'], + ['zomb fix [path]', 'delete dead code on a branch, gated by your build'], + ['zomb blueprint [path]', 'infer the rules this code already follows'], + ['zomb guard', 'Claude Code hook: blocks bad edits as they happen'], + ]; + const box = [ + faint(` ┌${'─'.repeat(W - 3)}┐`), + ` ${faint('│')} ${dim('COMMANDS')}${' '.repeat(W - 12)}${faint('│')}`, + ...commands.map(([c, what]) => { + const text = clip(what, W - 29); + return ` ${faint('│')} ${green(c.padEnd(23))}${text}${' '.repeat(W - 27 - text.length)}${faint('│')}`; + }), + faint(` └${'─'.repeat(W - 3)}┘`), + ]; + const flag = (f: string, what: string) => `${f.padEnd(18)}${dim(what)}`; + const groups: [string, string[]][] = [ + ['SCAN', [flag('--all', 'every finding'), flag('--since ', 'only what changed'), flag('--json', 'to-do for an agent'), flag('--markdown', 'a PR comment'), flag('--out ', 'HTML report path'), flag('--save-baseline', 'accept today'), flag('--fail-on ', 'the CI gate')]], + ['FIX', [flag('--dry-run', 'show the plan only'), flag('--no-checks', 'skip the build gate')]], + ['BLUEPRINT', [flag('--write', 'save the rules')]], + ['ENV', [flag('NO_COLOR=1', 'plain output'), flag('ZOMB_GUARD=off', 'pause the guard')]], + ]; + const block = ([title, lines]: [string, string[]]) => [bold(title), ...lines]; + let flags: string[]; + if (W >= 78) { + // two columns: scan on the left, everything else on the right + const left = block(groups[0]); + const right = groups.slice(1).flatMap((g, i) => [...(i ? [''] : []), ...block(g)]); + flags = Array.from({ length: Math.max(left.length, right.length) }, (_, i) => { + const l = left[i] || ''; + return ` ${l}${right[i] ? `${' '.repeat(39 - visible(l))}${right[i]}` : ''}`.trimEnd(); + }); + } else flags = groups.flatMap((g, i) => [...(i ? [''] : []), ...block(g).map((l) => ` ${l}`)]); + return ['', ...banner, '', row(dim('a health check for codebases written with AI'), version ? faint(`v${version}`) : '', 1), '', ...box, '', ...flags, ''].join('\n'); +} + // `zomb blueprint`: the proposed rules with their evidence, what breaks them today, and how to save them. export function renderBlueprint( rules: Rule[], today: { file: string; rule: string; why: string }[], { repo, width = 80, color = true, exists = false, write = false, file = '.zomb/blueprint.yml' }: { repo: string; width?: number; color?: boolean; exists?: boolean; write?: boolean; file?: string }, ): string { - const { W, bold, dim, yellow, green, cyan, n, plural, row, rule, frame } = ui({ width, color }); + const { W, bold, dim, yellow, green, underline, n, plural, row, head, dots } = ui({ width, color }); const TITLES = { libraries: 'one library per job', folders: 'where new files go', naming: 'how files are named', imports: 'no ../../../', api: 'routes that change data', files: 'size limit' }; - const L = ['', ...frame(`${bold('zomb blueprint')} ${repo}`, dim(plural(rules.length, 'rule')), [dim('the rules this code already follows, for you to approve')])]; + const L = ['', ...head(`${green('⬢ zomb blueprint')} ${bold(underline(repo))}`, dim(plural(rules.length, 'rule'))), row(dim('the rules this code already follows, for you to approve'), '', 1)]; for (const section of [...new Set(rules.map((r) => r.section))]) { - L.push('', rule(section.toUpperCase(), dim(TITLES[section]))); + L.push('', dots(bold(section), dim(TITLES[section]))); for (const r of rules.filter((x) => x.section === section)) { const k = new Set(today.filter((b) => b.rule === `${section}.${r.key}`).map((b) => b.file)).size; - const left = `${cyan(r.key.padEnd(12))}${bold(String(r.value))}`; + const left = `${green(r.key.padEnd(12))}${bold(String(r.value))}`; const right = `${dim(r.evidence)}${k ? yellow(` ✗ ${plural(k, 'file')}`) : ''}`; // evidence goes under the rule when both don't fit on one line - if (visible(left) + visible(right) + 6 <= W) L.push(row(left, right)); - else L.push(row(left), row(right, '', 14)); + if (visible(left) + visible(right) + 6 <= W) L.push(row(left, right, 3)); + else L.push(row(left, '', 3), row(right, '', 15)); } } const files = [...new Set(today.map((b) => b.file))]; if (today.length) { - L.push('', rule('BREAKS TODAY', yellow(plural(files.length, 'file')))); - for (const b of today.slice(0, 5)) L.push(row(`${yellow('✗')} ${b.why}`), row(dim(b.file), '', 4)); - if (today.length > 5) L.push(dim(` …and ${n(today.length - 5)} more`)); + L.push('', dots(bold('breaks today'), yellow(`${plural(today.length, 'break')} in ${plural(files.length, 'file')}`))); + for (const b of today.slice(0, 5)) L.push(row(`${yellow('✗')} ${b.why}`, '', 3), row(dim(b.file), '', 5)); + if (today.length > 5) L.push(dim(` …and ${n(today.length - 5)} more`)); } L.push(''); if (write && !exists) { - L.push(row(`${green('✓')} saved ${cyan(file)}`), row(dim('Commit it. Guard now enforces it on every agent edit, and the scan on every PR.'), '', 4)); - if (today.length) L.push(row(`${dim('accept the breaks you have today:')} ${cyan('zomb --save-baseline')}`, '', 4)); + L.push(row(`${green('✓')} saved ${green(file)}`, '', 1), row(dim('Commit it. Guard now enforces it on every agent edit, and the scan on every PR.'), '', 3)); + if (today.length) L.push(row(`${dim('accept the breaks you have today:')} ${green('zomb --save-baseline')}`, '', 3)); } else { - if (exists) L.push(row(`${yellow('!')} ${file} already exists: this is a fresh proposal to compare with it`)); - if (!write) L.push(row(`${cyan('zomb blueprint --write')} ${dim(`saves it to ${file}`)}`)); + if (exists) L.push(row(`${yellow('!')} ${file} already exists: this is a fresh proposal to compare with it`, '', 1)); + if (!write) L.push(row(`${green('zomb blueprint --write')} ${dim(`saves it to ${file}`)}`, '', 1)); } return `${L.join('\n')}\n`; } -// A one-line spinner on stderr while the scan runs; silent when stderr isn't a terminal. -export function spinner(label: string) { - if (!process.stderr.isTTY) return { step: () => {}, stop: () => {} }; - const frames = '⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏'; - const done: string[] = []; - let i = 0; +// While the scan runs: a dot matrix that ripples, and each check ticking off. Silent when stderr isn't a terminal. +export function loader({ title, label, steps, color = true }: { title: string; label: string; steps: [string, string][]; color?: boolean }) { + const out = process.stderr; + if (!out.isTTY) return { step: (_name: string) => {}, stop: () => {} }; + const c = (code: number | string, s: string) => (color ? `\x1b[${code}m${s}\x1b[0m` : s); + const LEVELS = [236, 22, 28, 35, 121]; // 256-colour greens, dark to bright + const started = Date.now(); + const done = new Map(); + const secs = () => `${((Date.now() - started) / 1000).toFixed(1)}s`; + let tick = 0; + let drawn = 0; const draw = () => { - const text = `${label}${done.length ? ` · ${done.join(' · ')}` : ''}`.slice(0, (process.stderr.columns || 80) - 3); - process.stderr.write(`\r\x1b[2K\x1b[36m${frames[i++ % frames.length]}\x1b[0m \x1b[2m${text}\x1b[0m`); + const cols = (out.columns || 80) - 1; + const size = Math.min(26, Math.max(cols - 44, 6)); + const matrix = (r: number) => + Array.from({ length: size }, (_, i) => { + const v = ((Math.sin(tick * 0.32 - i * 0.5 + r * 1.7) + 1) / 2) * 0.8 + Math.random() * 0.35; + return color ? `\x1b[38;5;${LEVELS[Math.min(4, Math.floor(v * 4))]}m■` : v > 0.6 ? '■' : '·'; + }).join('') + (color ? '\x1b[0m' : ''); + const lines = [ + ` ${matrix(0)} ${c(1, 'zomb')} ${title}`, + ` ${matrix(1)} ${c(2, `${label} · ${secs()}`)}`, + '', + ` ${c(2, '⬢')} ${c(1, 'Combing')} through ${steps.length} checks ${c(2, `${done.size} of ${steps.length}`)}`, + ...steps.map(([name, what]) => (done.has(name) ? ` ${c(90, `☒ ${name.padEnd(12)}${what.padEnd(22)}${done.get(name)}`)}` : ` ${c(1, '▣')} ${name.padEnd(12)}${c(2, what)}`)), + ].map((l) => (visible(l) > cols ? l.replace(ANSI, '').slice(0, cols) : l)); + // redraw in place: back to the first line, then every line cleared and rewritten + out.write(`${drawn ? `\x1b[${drawn}F` : '\x1b[?25l'}${lines.map((l) => `\x1b[2K${l}`).join('\n')}\n`); + drawn = lines.length; + tick++; }; - const timer = setInterval(draw, 80); - const started = Date.now(); + const showCursor = () => out.write('\x1b[?25h'); + process.once('exit', showCursor); + const timer = setInterval(draw, 90); draw(); return { - step: (name) => done.push(`${name} ✓`), - // leave one quiet line behind instead of a half-cleared spinner - stop: () => (clearInterval(timer), process.stderr.write(`\r\x1b[2K\x1b[32m✓\x1b[0m \x1b[2m${label.replace('scanning', 'scanned')} in ${((Date.now() - started) / 1000).toFixed(1)}s\x1b[0m\n`)), + step: (name: string) => void done.set(name, secs()), + // leave one quiet line behind instead of the whole block + stop: () => { + clearInterval(timer); + out.write(`\x1b[${drawn}F\x1b[J ${c(32, '✓')} ${c(2, `${label.replace('scanning', 'scanned')} in ${secs()}`)}\n`); + showCursor(); + }, }; } diff --git a/test/e2e.test.ts b/test/e2e.test.ts index 43855d6..78636a2 100644 --- a/test/e2e.test.ts +++ b/test/e2e.test.ts @@ -79,6 +79,13 @@ test('zomb --json finds every planted mistake', () => { run('--save-baseline'); assert.equal(run('--fail-on', 'high', '--json').code, 0, 'known problems pass once they are in the baseline'); + // `zomb show ` explains the finding the scan numbered n + const show = (...args) => execFileSync(process.execPath, [cli, 'show', ...args, dir], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }); + const out1 = show('1'); + assert.ok(out1.includes(JSON.parse(run('--json').out).tasks[0].title) && out1.includes('finding 1 of'), out1); + assert.throws(() => show('999'), /pick 1 to/); + assert.throws(() => show('one'), /takes a finding number/); + // A change that adds a new key and takes shortcuts. writeFileSync(path.join(dir, 'lib/github.ts'), `export const token = "${['ghp', 'Zq9rT4vWxyKm2Lp8Nc1Hs6Kd3Vb7Qe5Tr0Uy'].join('_')}";\n`); writeFileSync(path.join(dir, 'lib/db.ts'), files['lib/db.ts'] + '// @ts-ignore\nexport const n: number = "1";\n'); diff --git a/test/score.test.ts b/test/score.test.ts index cf2fd05..e99e94b 100644 --- a/test/score.test.ts +++ b/test/score.test.ts @@ -171,7 +171,8 @@ test('scopeTo keeps only what the change touched', async () => { }); test('terminal output never wraps, at any width', async () => { - const { render, visible } = await import('../src/terminal.ts'); + const { render, renderHelp, visible } = await import('../src/terminal.ts'); + const { toTasks } = await import('../src/tasks.ts'); const long = 'src/components/features/dashboard/analytics/really-long-folder-name/SomeVeryLongComponentName.tsx'; const data = { repo: 'a-repo-with-a-fairly-long-name', files: 1234, lines: 123456, @@ -181,9 +182,13 @@ test('terminal output never wraps, at any width', async () => { sprawl: { dupes: [{ lines: 30, a: { file: long }, b: { file: long } }], names: [{ name: 'formatCurrencyWithLocaleAndFallbackSymbol', files: [long, long] }], versions: [{ path: long, original: long }, { path: 'components/ov2/', files: 75 }], overlaps: [{ job: 'icon sets', libraries: [{ name: '@phosphor-icons/react', files: 40 }, { name: 'react-icons', files: 3 }, { name: '@heroicons/react', files: 1 }] }] }, architecture: { cycles: [[long, long, long, long]], big: [{ file: long, lines: 1727 }], deep: [{ file: long, count: 3 }], shared: ['src/lib', 'src/utils', 'app/helpers', 'packages/shared/src/core'], naming: [] }, }; - const tasks = [{ safe: true }, { safe: false }]; + const tasks = toTasks(data as unknown as ScanData); + const days = new Map([[new Date().toLocaleDateString('en-CA'), 900], ['2026-01-02', 40]]); for (const width of [50, 60, 80, 120]) { - const lines = render(data as unknown as ScanData, tasks as Task[], { width, color: true, all: true, recent: [{ month: '2026-09', added: 34628, deleted: 15120 }], out: `/Users/someone/projects/${long}/.zomb/report.html`, failOn: 'high', failing: [1] }); + const lines = [ + ...render(data as unknown as ScanData, tasks, { width, color: true, all: true, recent: [{ month: '2026-09', added: 34628, deleted: 15120 }], days, out: `/Users/someone/projects/${long}/.zomb/report.html`, failOn: 'high', failing: [1] }), + ...renderHelp({ width, version: '1.0.0' }).split('\n'), + ]; const widest = Math.max(...lines.map(visible)); assert.ok(widest <= Math.max(48, Math.min(width - 1, 80)), `width ${width}: a line is ${widest} columns`); assert.ok(lines.every((l) => !/ $/.test(l.replace(/\x1b\[[0-9;]*m/g, ''))), 'no trailing spaces'); From af4b15340f195347330c79cad793b88e19ca9511 Mon Sep 17 00:00:00 2001 From: Diip3sh Date: Tue, 13 Oct 2026 12:38:49 +0530 Subject: [PATCH 2/7] Read production traffic: routes no request reached zomb --traffic reads access logs, JSON lines or arrays (Vercel drains and most log shippers) and plain "/path count" exports, .gz or not, and keeps only paths, counts and dates. Each request is credited to the route Next.js would serve it from, so /blog/hello keeps /blog/[slug] alive and /blog/new stays its own route; a locale prefix is ignored. With traffic, a route nobody requested is the finding: "no links and no traffic" (delete-route) when nothing in the repo points to it, "no traffic" when something still links to it. API-only repos are checked too. A file with no readable requests fails rather than calling every route dead, and under a week of traffic gets a warning. Without the flag, every output is what it was. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- README.md | 14 ++- skills/zomb-clean/SKILL.md | 4 +- src/cli.ts | 25 ++++- src/report.ts | 17 +++- src/signals.ts | 13 +-- src/tasks.ts | 13 ++- src/terminal.ts | 8 +- src/traffic.ts | 183 +++++++++++++++++++++++++++++++++++++ src/types.ts | 5 +- test/e2e.test.ts | 52 +++++++++++ test/traffic.test.ts | 77 ++++++++++++++++ 11 files changed, 388 insertions(+), 23 deletions(-) create mode 100644 src/traffic.ts create mode 100644 test/traffic.test.ts diff --git a/README.md b/README.md index 04108c4..b736c10 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,7 @@ zomb show 3 # finding 3 from the list in full: where, why, how to zomb --json # an ordered to-do list for an AI agent or CI zomb --since main # only what your branch changed, plus the shortcuts it took zomb --since HEAD # only your uncommitted work +zomb --traffic access.log # the routes no production request reached ``` ## Compared with 336 public repos @@ -96,6 +97,17 @@ It refuses to run on uncommitted work, and it never touches scripts you may run With `--since`, zomb also reads the diff for the ways agents make checks pass without fixing anything: new `@ts-ignore` and `eslint-disable`, `.skip` and `.only` tests, deleted tests and assertions, branches on `NODE_ENV === 'test'`, `as any`, and empty `catch` blocks. +## Dead routes, from real traffic + +Without traffic, a page or API route that nothing in the repo links to is only a guess, so zomb asks about it. Give it your production requests and it knows: + +```bash +zomb --traffic access.log # nginx or Apache logs +zomb --traffic drain.ndjson --traffic last-month.log.gz # several files, gzipped or not +``` + +It reads access logs, JSON lines or arrays (Vercel log drains, most log shippers, anything with a `path` or `url`), and plain `/path count` lines like an analytics export. Every route no request reached becomes a finding: "no links and no traffic" when nothing in the repo points to it either, "no traffic" when something still links to it. A dynamic route counts the requests it serves (`/blog/hello` keeps `/blog/[slug]` alive), and a locale prefix like `/fr/` is ignored. Only paths, counts and dates are read. Less than a week of traffic gets a warning, since a page used once a month would look dead. [zomb Cloud](cloud/README.md) collects the traffic for you from a log drain. + ## Block new problems in CI Existing debt shouldn't fail every build. Save it once as a baseline, commit it, and only new findings fail: @@ -156,7 +168,7 @@ Any other agent can follow [skills/zomb-guard/SKILL.md](skills/zomb-guard/SKILL. **Zombie code:** code that's in the codebase but not in use - Files nothing imports or names, and files only their own tests keep alive - Unused npm packages, and exports nothing imports -- Pages and API routes nothing in the repo links to (listed as "maybe", check your analytics) +- Pages and API routes nothing in the repo links to (listed as "maybe"), or, with `--traffic`, that no production request reached **Sprawl:** the codebase only grows - Lines added vs deleted each month diff --git a/skills/zomb-clean/SKILL.md b/skills/zomb-clean/SKILL.md index 8a93996..58cb408 100644 --- a/skills/zomb-clean/SKILL.md +++ b/skills/zomb-clean/SKILL.md @@ -51,7 +51,7 @@ If `zomb fix` is not available, do the same by hand: batches of up to 10, checks Tasks with area `blueprint` break a rule the team approved in `.zomb/blueprint.yml`: switch to the named library, move or rename the file, import through the alias, or add the auth check. Never edit the blueprint to make a break go away; ask the user if a rule looks wrong. -Libraries doing the same job, versioned copies, duplicated code, "maybe" routes, import cycles and oversized files all change how the code is organised. List them with one line each and ask which to do. Do one at a time, with the checks after each. +Libraries doing the same job, versioned copies, duplicated code, "maybe" routes and routes with no traffic (`delete-route`), import cycles and oversized files all change how the code is organised. List them with one line each and ask which to do. Do one at a time, with the checks after each. ## 6. Report @@ -59,7 +59,7 @@ Run `zomb --json` again and tell the user: - before → after: security issues, lines of zombie code, unused packages - the commits you made on the branch -- what only they can do: rotate the keys you moved, decide the "maybe" routes, review and merge the branch +- what only they can do: rotate the keys you moved, decide the "maybe" and no-traffic routes, review and merge the branch ## Rules diff --git a/src/cli.ts b/src/cli.ts index 03935c4..e80e83d 100755 --- a/src/cli.ts +++ b/src/cli.ts @@ -6,7 +6,8 @@ import { createRequire } from 'node:module'; import path from 'node:path'; import { parseArgs, promisify } from 'node:util'; import { isAgent, parseBlame, testsOnly, isTest, growth, versionSprawl, overlaps, sameNames, cycles, sharedFolders, namingStyles } from './score.ts'; -import { parseAll, dependents, touches, clones, orphanRoutes } from './signals.ts'; +import { parseAll, dependents, touches, clones, routeLinks } from './signals.ts'; +import { readTraffic, routeHits, summarize } from './traffic.ts'; import { scanRepo, npmAudit, openRoute, hasAuthSignal, securityFindings } from './security.ts'; import { renderReport } from './report.ts'; import { toTasks, fingerprint, toMarkdown } from './tasks.ts'; @@ -36,6 +37,7 @@ try { markdown: { type: 'boolean', default: false }, all: { type: 'boolean', default: false }, since: { type: 'string' }, + traffic: { type: 'string', multiple: true }, 'fail-on': { type: 'string' }, 'save-baseline': { type: 'boolean', default: false }, 'dry-run': { type: 'boolean', default: false }, @@ -67,6 +69,8 @@ const ZOMB_IGNORE = '*\n!baseline.json\n!blueprint.yml\n'; const target = path.resolve(positionals[0] || '.'); if (!existsSync(target)) fail(`${target} does not exist`); if (opts['fail-on'] && !LEVELS[opts['fail-on']]) fail(`--fail-on takes high, medium or low, not "${opts['fail-on']}"`); +const trafficFiles = (opts.traffic || []).map((f) => path.resolve(f)); +for (const f of trafficFiles) if (!existsSync(f)) fail(`--traffic: ${f} does not exist`); const root = (await sh('git', ['rev-parse', '--show-toplevel'], target).catch(() => fail(`${target} is not inside a git repo`))).trim(); async function knip() { @@ -125,21 +129,22 @@ const all = [...new Set((await sh('git', ['ls-files', '--cached', '--others', '- const tracked = new Set((await sh('git', ['ls-files'], root)).split('\n').filter(Boolean)); const files = all.filter((f) => CODE.test(f) && !/\.d\.[cm]?ts$/.test(f)); if (!files.length) fail(`no JS/TS files tracked in ${root}`); -const CHECKS: [string, string][] = [['imports', 'what nothing uses'], ['history', 'who wrote what'], ['code', `parse ${files.length} files`], ['copy-paste', 'repeated blocks'], ['routes', 'API routes and auth'], ['secrets', 'keys and .env files'], ['packages', 'npm audit']]; +const CHECKS: [string, string][] = [['imports', 'what nothing uses'], ['history', 'who wrote what'], ['code', `parse ${files.length} files`], ['copy-paste', 'repeated blocks'], ['routes', 'API routes and auth'], ['secrets', 'keys and .env files'], ['packages', 'npm audit'], ...(trafficFiles.length ? [['traffic', 'production requests'] as [string, string]] : [])]; const spin = opts.json || opts.markdown ? { step() {}, stop() {} } : loader({ title: path.basename(root), label: `scanning ${files.length} files`, steps: CHECKS, color: !process.env.NO_COLOR }); if (!process.stderr.isTTY) log(`scanning ${files.length} JS/TS files in ${root}`); const track = (name, p) => p.then((v) => (spin.step(name), v)); const middleware = files.find((f) => /(^|\/)middleware\.[cm]?[jt]s$/.test(f)); -const [knipOut, { agentOf, history }, parsed, copies, orphans, scan, audit, middlewareSrc] = await Promise.all([ +const [knipOut, { agentOf, history }, parsed, copies, routes, scan, audit, middlewareSrc, traffic] = await Promise.all([ track('imports', knip()), track('history', commits()), track('code', parseAll(root, files)), track('copy-paste', clones(root, files)), - track('routes', orphanRoutes(root, files)), + track('routes', routeLinks(root, files)), track('secrets', scanRepo(root, all, tracked)), track('packages', npmAudit(root)), middleware ? readFile(path.join(root, middleware), 'utf8') : '', + trafficFiles.length ? track('traffic', readTraffic(trafficFiles).catch((e) => (spin.stop(), fail(`--traffic: ${String(why(e).message).split('\n')[0]}`)))) : null, ]); // ---------- zombie code ---------- @@ -161,6 +166,8 @@ for (;;) { const zombieFiles = [...unused, ...onlyTests].filter((f) => !refs.get(f) && !live.has(f)); spin.stop(); if (knipOut.error) process.stderr.write(`${yellow('!')} Knip failed, so zombie detection is skipped: ${knipOut.error}\n`); +// a file zomb can't read would make every route look unvisited +if (traffic && !traffic.requests) fail(`--traffic: no requests found in ${trafficFiles.map((f) => path.relative(process.cwd(), f) || f).join(', ')}. zomb reads access logs, JSON lines and "/path count" lines.`); const byAgent = await authorship(zombieFiles, agentOf); const zombies = zombieFiles .map((f) => { @@ -175,7 +182,13 @@ const deadExports = knipOut.issues .filter((i) => !unused.has(i.file) && i.exports.length + i.types.length) .map((i) => ({ file: i.file, names: [...i.exports, ...i.types].map((e) => e.name) })) .sort((a, b) => b.names.length - a.names.length); -const maybe = [...orphans].map(([file, url]) => ({ file, url, lines: parsed.get(file).lines })); +// Without traffic, a route nothing links to is a guess. With it, a route no request reached is the finding, +// and the links only say how sure: API-only repos and linked pages count too. +const routeList = [...routes].map(([file, r]) => ({ file, url: r.url, lines: parsed.get(file).lines, linked: r.linked })); +const served = traffic && routeHits(routeList.map((r) => r.url), traffic); +const maybe = served ? routeList.filter((r) => !served.get(r.url)) : routeList.filter((r) => r.linked === false).map(({ file, url, lines }) => ({ file, url, lines })); +const trafficSummary = traffic && summarize(traffic, routeList.length, maybe.length); +if (trafficSummary && trafficSummary.days !== null && trafficSummary.days < 7) process.stderr.write(`${yellow('!')} the traffic covers ${trafficSummary.days === 1 ? '1 day' : `${trafficSummary.days} days`}, so routes used weekly or monthly will look unused\n`); // ---------- security ---------- const touchesOf = touches(parsed); @@ -226,6 +239,7 @@ let data: ScanData = { blueprint: rules && { broken }, // only in a whole-repo scan: a diff has no history to compare benchmark: opts.since ? null : benchmark(history, months), + traffic: trafficSummary, }; // --since: report only what this change touched, plus the shortcuts it took if (opts.since) { @@ -287,6 +301,7 @@ if (opts.json) { // the denominators a dashboard needs to show shares rather than raw counts lines: data.lines, deletedPer100: data.benchmark?.deletedPer100 ?? null, + ...(data.traffic && { traffic: data.traffic }), }; console.log(JSON.stringify({ repo: data.repo, commit: data.commit, since: data.since || null, baseline: Boolean(baseline), failing: failing.length, summary, tasks }, null, 2)); done(); diff --git a/src/report.ts b/src/report.ts index 8dfec1b..3bf2ec3 100644 --- a/src/report.ts +++ b/src/report.ts @@ -1,3 +1,4 @@ +import { span } from './traffic.ts'; import type { Clone, Month, ScanData, Shortcut } from './types.ts'; const esc = (s: unknown) => String(s).replace(/[&<>"']/g, (c) => `&#${c.charCodeAt(0)};`); @@ -25,14 +26,22 @@ function shortcutsSection(shortcuts: Shortcut[] | undefined) { ${shortcuts.length ? `
    ${shortcuts.map((c) => `
  • ${c.severity}
    ${esc(c.kind)} ${code(c.line ? `${c.file}:${c.line}` : c.file)}
    • `).join('')}
` : '

None found.

'}`; } -function zombieSection({ files, packages, exports, maybe }: ScanData['zombie'], knipError?: string) { +function routesSection(maybe: ScanData['zombie']['maybe'], traffic: ScanData['traffic']) { + if (!traffic) { + return maybe.length ? `

Maybe zombie ${maybe.length}

Pages and API routes nothing in the repo links to or calls. Outside links or other services may still use them, so check your analytics before deleting.

    ${maybe.map((m) => `
  • ${code(m.url)} ${esc(m.file)} · ${n(m.lines)} lines
    • `).join('')}
` : ''; + } + return `

No traffic ${maybe.length} of ${plural(traffic.routes, 'route')}

Pages and API routes no request reached ${esc(span(traffic))}. The ones marked "no links" aren't linked to or called from anywhere in the repo either.

+ ${maybe.length ? `
    ${maybe.map((m) => `
  • ${code(m.url)} ${esc(m.file)} · ${n(m.lines)} lines${m.linked === false ? ' · no links' : ''}
    • `).join('')}
` : '

Every route got requests.

'}`; +} + +function zombieSection({ files, packages, exports, maybe }: ScanData['zombie'], knipError: string | undefined, traffic: ScanData['traffic']) { if (knipError) return `

Zombie code

Skipped: ${esc(knipError)}

`; const lines = files.reduce((s, f) => s + f.lines, 0); return `

Zombie code

${files.length ? `${plural(files.length, 'file')} (${n(lines)} lines) sit in your codebase but aren't in use. Delete them and nothing breaks.` : 'Every file is in use.'}

${files.length ? `
    ${files.slice(0, 50).map((f) => `
  • ${code(f.path)} ${n(f.lines)} lines

    ${esc(f.why)}${f.share >= 0.5 ? `. ${Math.round(f.share * 100)}% written by ${esc(f.agents.join(', '))}` : ''}

    • `).join('')}
${more(files.length, 50)}` : ''} ${packages.length ? `

Unused packages ${packages.length}

Installed but never imported. Every one is extra install time, bundle risk and attack surface.

${packages.map((p) => `${code(p.name)}${p.dev ? ' dev' : ''}`).join(' ')}

` : ''} - ${maybe.length ? `

Maybe zombie ${maybe.length}

Pages and API routes nothing in the repo links to or calls. Outside links or other services may still use them, so check your analytics before deleting.

    ${maybe.map((m) => `
  • ${code(m.url)} ${esc(m.file)} · ${n(m.lines)} lines
    • `).join('')}
` : ''} + ${routesSection(maybe, traffic)} ${exports.length ? `

Dead code inside live files ${n(exports.reduce((s, e) => s + e.names.length, 0))} exports

Functions, components and types that are exported but never imported anywhere.

    ${exports.slice(0, 15).map((e) => `
  • ${code(e.file)}

    ${esc(e.names.slice(0, 8).join(', '))}${e.names.length > 8 ? ` and ${e.names.length - 8} more` : ''}

    • `).join('')}
${more(exports.length, 15)}` : ''}
`; } @@ -91,7 +100,7 @@ function benchmarkSection(bm: ScanData['benchmark']) {

Median commit here adds ${n(bm.medianCommit)} lines; a human commit in the study adds ${n(bm.humanMedianCommit)}.

`; } -export function renderReport({ repo, commit, date, files, lines, knipError, zombie, security, sprawl, architecture, shortcuts, since, blueprint, benchmark }: ScanData): string { +export function renderReport({ repo, commit, date, files, lines, knipError, zombie, security, sprawl, architecture, shortcuts, since, blueprint, benchmark, traffic }: ScanData): string { const high = security.findings.filter((f) => f.severity === 'high').length; const zLines = zombie.files.reduce((s, f) => s + f.lines, 0); const added = sprawl.recent.reduce((s, m) => s + m.added, 0); @@ -126,7 +135,7 @@ ${card('architecture', n(architecture.cycles.length), 'import cycles', `${plural ${securitySection(security)} ${blueprintSection(blueprint)} ${shortcutsSection(shortcuts)} -${zombieSection(zombie, knipError)} +${zombieSection(zombie, knipError, traffic)} ${sprawlSection(sprawl)} ${benchmarkSection(benchmark)} ${architectureSection(architecture)} diff --git a/src/signals.ts b/src/signals.ts index 3a1e6e8..6e3323b 100644 --- a/src/signals.ts +++ b/src/signals.ts @@ -98,21 +98,22 @@ export function touches(parsed: Map): Map { return out; } -// Page/API routes that nothing in the repo links to or calls -> Map(path -> url). -// ponytail: static search for the URL's fixed prefix; production traffic (cloud) is the real answer -export async function orphanRoutes(root: string, files: string[]): Promise> { +// Every page/API route -> Map(path -> { url, linked }): whether anything in the repo links to or calls it. +// `linked` is null for an API-only repo, where nothing inside is supposed to call its routes. +// The static search for the URL's fixed prefix is a guess; production traffic (--traffic) is the real answer. +export async function routeLinks(root: string, files: string[]): Promise> { const out = new Map(); - // A repo with no pages besides a home page is an API for other apps: nothing inside it is supposed to call its routes. const hasPages = files.some((f) => routeOf(f)?.kind === 'page'); await Promise.all( files.map(async (file) => { const route = routeOf(file); - if (!route || (route.kind === 'api' && !hasPages)) return; + if (!route) return; + if (route.kind === 'api' && !hasPages) return out.set(file, { url: route.url, linked: null }); const prefix = route.prefix.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); // a quote, backtick, ( or } right before the path; /, ?, #, a quote or line end right after const re = `["'\`(}]${prefix}([/?#"'\`]|$)`; const hits = await promisify(execFile)('git', ['grep', '-l', '--untracked', '-E', '-e', re, '--', '.', `:!${file}`, ':!.zomb', ':!*zomb-report*.html'], { cwd: root }).then((r) => r.stdout, () => ''); - if (!hits.trim()) out.set(file, route.url); + out.set(file, { url: route.url, linked: Boolean(hits.trim()) }); }), ); return out; diff --git a/src/tasks.ts b/src/tasks.ts index ed45a82..e79eabd 100644 --- a/src/tasks.ts +++ b/src/tasks.ts @@ -1,10 +1,11 @@ // Findings -> one ordered to-do list that an AI agent (or a person) can work through. // `safe` = mechanical and checkable by a build: an agent may do it without asking. Everything else needs a human yes. +import { span } from './traffic.ts'; import type { ScanData, Severity, Task } from './types.ts'; const plural = (k: number, word: string) => `${k} ${word}${k === 1 ? '' : 's'}`; -export function toTasks({ security, zombie, sprawl, architecture, shortcuts = [], blueprint }: ScanData): Task[] { +export function toTasks({ security, zombie, sprawl, architecture, shortcuts = [], blueprint, traffic }: ScanData): Task[] { const tasks: Task[] = []; const add = (t: Omit) => tasks.push({ id: tasks.length + 1, ...t }); @@ -52,7 +53,15 @@ export function toTasks({ security, zombie, sprawl, architecture, shortcuts = [] if (zombie.packages.length) add({ area: 'zombie', severity: 'low', action: 'uninstall', title: `Uninstall ${plural(zombie.packages.length, 'unused package')}`, where: zombie.packages.map((p) => p.name).join(' '), how: 'Nothing imports them. Remove them with your package manager, then run the build.', safe: true }); for (const e of zombie.exports) add({ area: 'zombie', severity: 'low', action: 'remove-exports', title: `Remove ${plural(e.names.length, 'unused export')} from ${e.file}`, where: e.file, how: `Nothing imports ${e.names.join(', ')}. Delete them, or drop the export keyword if the file uses them itself.`, safe: true }); - for (const m of zombie.maybe) add({ area: 'zombie', severity: 'low', action: 'review', title: `Is ${m.url} still used?`, where: m.file, how: 'Nothing in the repo links to or calls it. Ask the owner or check analytics before deleting.', safe: false }); + for (const m of zombie.maybe) { + if (!traffic) { + add({ area: 'zombie', severity: 'low', action: 'review', title: `Is ${m.url} still used?`, where: m.file, how: 'Nothing in the repo links to or calls it. Ask the owner or check analytics before deleting.', safe: false }); + continue; + } + const seen = `No request reached it ${span(traffic)}`; + if (m.linked === false) add({ area: 'zombie', severity: 'low', action: 'delete-route', title: `Delete ${m.url}: no links and no traffic`, where: m.file, how: `${seen}, and nothing in the repo links to or calls it. Delete the route and anything only it uses, then run the build and tests.`, safe: false }); + else add({ area: 'zombie', severity: 'low', action: 'review', title: `Is ${m.url} still used? No traffic`, where: m.file, how: `${seen}${m.linked ? ', though the repo still links to it' : ''}. Check with whoever owns it before deleting${m.linked ? ', and remove the links with it' : ''}.`, safe: false }); + } for (const o of sprawl.overlaps) { const [keep, ...rest] = o.libraries; diff --git a/src/terminal.ts b/src/terminal.ts index b19c5f4..5817a15 100644 --- a/src/terminal.ts +++ b/src/terminal.ts @@ -1,5 +1,6 @@ // The terminal look for every zomb command. Fits the window: nothing wraps, long paths shorten from the left, numbers line up on the right. // One visual language: ⬢ marks zomb at work, ● ◐ ○ are high, medium and low, ☒ ▣ ☐ are done, running and next. +import { span } from './traffic.ts'; import type { Month, Rule, ScanData, Severity, Task } from './types.ts'; const ANSI = /\x1b\[[0-9;]*m/g; @@ -107,7 +108,10 @@ export function render( security: audit.skipped ? [` ${dim(clip(`packages not checked: ${audit.skipped}`, W - 3))}`] : [], shortcuts: [], blueprint: [], - zombie: byAgent ? [` ${dim(`${byAgent} of ${plural(z.files.length, 'dead file')} written by an AI agent`)}`] : [], + zombie: [ + ...(byAgent ? [` ${dim(`${byAgent} of ${plural(z.files.length, 'dead file')} written by an AI agent`)}`] : []), + ...(data.traffic ? [` ${dim(clip(`traffic ${span(data.traffic)}: ${n(data.traffic.unvisited)} of ${plural(data.traffic.routes, 'route')} got none`, W - 3))}`] : []), + ], sprawl: added && !data.since ? [` ${dim(clip(`+${n(added)} −${n(deleted)} lines in 3 months${data.benchmark ? '' : ` · ${Math.round((deleted / added) * 100)} deleted per 100 added`}`, W - 3))}`] : [], architecture: [...(a.shared.length > 1 ? wrap(dim('shared code in'), a.shared, 3) : []), ...(a.deep.length ? [` ${dim(`${plural(a.deep.length, 'file')} with ../../../ imports`)}`] : [])], }; @@ -252,7 +256,7 @@ export function renderHelp({ width = 80, color = true, version = '' }: { width?: ]; const flag = (f: string, what: string) => `${f.padEnd(18)}${dim(what)}`; const groups: [string, string[]][] = [ - ['SCAN', [flag('--all', 'every finding'), flag('--since ', 'only what changed'), flag('--json', 'to-do for an agent'), flag('--markdown', 'a PR comment'), flag('--out ', 'HTML report path'), flag('--save-baseline', 'accept today'), flag('--fail-on ', 'the CI gate')]], + ['SCAN', [flag('--all', 'every finding'), flag('--since ', 'only what changed'), flag('--json', 'to-do for an agent'), flag('--markdown', 'a PR comment'), flag('--out ', 'HTML report path'), flag('--traffic ', 'routes nobody hits'), flag('--save-baseline', 'accept today'), flag('--fail-on ', 'the CI gate')]], ['FIX', [flag('--dry-run', 'show the plan only'), flag('--no-checks', 'skip the build gate')]], ['BLUEPRINT', [flag('--write', 'save the rules')]], ['ENV', [flag('NO_COLOR=1', 'plain output'), flag('ZOMB_GUARD=off', 'pause the guard')]], diff --git a/src/traffic.ts b/src/traffic.ts new file mode 100644 index 0000000..1e7be7d --- /dev/null +++ b/src/traffic.ts @@ -0,0 +1,183 @@ +// Production traffic: which page and API routes still get requests. Reads access logs (nginx, Apache), JSON +// lines or arrays (Vercel drains, most log shippers, zomb Cloud) and plain "/path count" lines. Keeps only +// paths, counts and the time range: nothing about who sent the request. +import { createReadStream } from 'node:fs'; +import { readFile } from 'node:fs/promises'; +import { createInterface } from 'node:readline'; +import { createGunzip, gunzipSync } from 'node:zlib'; + +export type Traffic = { hits: Map; requests: number; from: number | null; to: number | null }; +export type TrafficSummary = { requests: number; from: string | null; to: string | null; days: number | null; routes: number; unvisited: number }; +type Hit = { path: string; count: number; at: number | null }; + +const MONTHS = { Jan: 0, Feb: 1, Mar: 2, Apr: 3, May: 4, Jun: 5, Jul: 6, Aug: 7, Sep: 8, Oct: 9, Nov: 10, Dec: 11 }; +const LOG_REQUEST = /"(?:GET|POST|PUT|PATCH|DELETE|HEAD|OPTIONS) (\S+) HTTP\/[\d.]+"/; +const LOG_TIME = /\[(\d{2})\/(\w{3})\/(\d{4}):(\d{2}):(\d{2}):(\d{2}) ([+-])(\d{2})(\d{2})\]/; + +/** "/blog/post/?utm=x#top" or "https://site.com/blog/post" -> "/blog/post"; null when it isn't a path. */ +export function normalPath(raw: unknown): string | null { + if (typeof raw !== 'string' || !raw) return null; + let p = raw.trim().replace(/^[a-z][a-z0-9+.-]*:\/\/[^/]*/i, '').split(/[?#]/)[0]; + if (!p.startsWith('/')) return null; + try { + p = decodeURI(p); + } catch { + // a stray % is still a path + } + p = p.replace(/\/{2,}/g, '/'); + return p.length > 1 ? p.replace(/\/+$/, '') : p; +} + +// a number of seconds or milliseconds, or anything Date.parse reads +function time(v: unknown): number | null { + if (typeof v === 'number' && Number.isFinite(v)) return v < 1e11 ? v * 1000 : v; + if (typeof v === 'string' && v) { + if (/^\d+(\.\d+)?$/.test(v)) return time(Number(v)); + const t = Date.parse(v); + return Number.isNaN(t) ? null : t; + } + return null; +} + +function fromObject(o: any): Hit | null { + if (!o || typeof o !== 'object') return null; + const path = [o.path, o.pathname, o.requestPath, o.request_path, o.proxy?.path, o.request?.path, o.request?.url, o.req?.url, o.http?.path, o.url, o.page] + .map(normalPath) + .find(Boolean); + if (!path) return null; + const count = Number(o.count ?? o.hits ?? o.requests ?? o.visits ?? o.pageviews ?? 1); + const at = [o.timestamp, o.time, o['@timestamp'], o.ts, o.date, o.proxy?.timestamp].map(time).find((t) => t !== null) ?? null; + return { path, count: Number.isFinite(count) && count >= 0 ? count : 1, at }; +} + +/** One line of a log or export -> the request it records, or null when the line isn't one. */ +export function readLine(line: string): Hit | null { + const text = line.trim(); + if (!text) return null; + if (text.startsWith('{')) { + try { + return fromObject(JSON.parse(text)); + } catch { + return null; + } + } + const req = text.match(LOG_REQUEST); + if (req) { + const path = normalPath(req[1]); + if (!path) return null; + const t = text.match(LOG_TIME); + const at = t ? Date.UTC(Number(t[3]), MONTHS[t[2]] ?? 0, Number(t[1]), Number(t[4]), Number(t[5]), Number(t[6])) - (t[7] === '-' ? -1 : 1) * (Number(t[8]) * 60 + Number(t[9])) * 60_000 : null; + return { path, count: 1, at }; + } + // "/path", "/path 12", "/path,12": an analytics export or a list someone made by hand + if (text.startsWith('/')) { + const [raw, n] = text.split(/[\s,;\t]+/); + const path = normalPath(raw); + const count = Number(n); + return path ? { path, count: n !== undefined && Number.isFinite(count) && count >= 0 ? count : 1, at: null } : null; + } + return null; +} + +/** Add what one file (or a webhook body) says to `into`. A file that is one JSON array is read whole. */ +export function addText(into: Traffic, text: string) { + const trimmed = text.trimStart(); + if (trimmed.startsWith('[')) { + try { + for (const o of JSON.parse(trimmed)) add(into, fromObject(o)); + return; + } catch { + // not one array after all: fall through to lines + } + } + for (const line of text.split('\n')) add(into, readLine(line)); +} + +function add(into: Traffic, hit: Hit | null) { + if (!hit) return; + into.hits.set(hit.path, (into.hits.get(hit.path) || 0) + hit.count); + into.requests += hit.count; + if (hit.at !== null) { + into.from = into.from === null ? hit.at : Math.min(into.from, hit.at); + into.to = into.to === null ? hit.at : Math.max(into.to, hit.at); + } +} + +export const emptyTraffic = (): Traffic => ({ hits: new Map(), requests: 0, from: null, to: null }); + +const open = (file: string) => (file.endsWith('.gz') ? createReadStream(file).pipe(createGunzip()) : createReadStream(file)); + +/** Read every file (plain or .gz) into one tally. Large logs are streamed line by line. */ +export async function readTraffic(files: string[]): Promise { + const into = emptyTraffic(); + for (const file of files) { + // a JSON array has to be parsed whole; everything else streams + if ((await firstChar(file)) === '[') { + addText(into, file.endsWith('.gz') ? gunzipSync(await readFile(file)).toString('utf8') : await readFile(file, 'utf8')); + continue; + } + for await (const line of createInterface({ input: open(file), crlfDelay: Infinity })) add(into, readLine(String(line))); + } + return into; +} +function firstChar(file: string) { + return new Promise((resolve, reject) => { + const raw = createReadStream(file); + const s = file.endsWith('.gz') ? raw.pipe(createGunzip()) : raw; + let seen = ''; + const done = (ch: string) => (resolve(ch), s.destroy(), raw.destroy()); + s.on('data', (c: Buffer) => { + seen += c.toString('utf8'); + if (seen.trim()) done(seen.trimStart()[0]); + }); + s.once('error', reject); + s.once('end', () => done('')); + }); +} + +// A Next.js route URL -> a regex for the request paths it serves, and how specific it is. +// /blog/[slug] serves /blog/x, /docs/[...slug] one or more segments, /shop/[[...slug]] zero or more. +function pattern(url: string): { re: RegExp; rank: number[] } { + const rank: number[] = []; + let src = ''; + for (const seg of url.split('/').filter(Boolean)) { + if (/^\[\[\.\.\..+\]\]$/.test(seg)) (src += '(?:/[^/]+)*'), rank.push(0); + else if (/^\[\.\.\..+\]$/.test(seg)) (src += '(?:/[^/]+)+'), rank.push(1); + else if (/^\[.+\]$/.test(seg)) (src += '/[^/]+'), rank.push(2); + else (src += `/${seg.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}`), rank.push(3); + } + return { re: new RegExp(`^${src || '/'}$`), rank }; +} +// Next.js picks the most specific route: /blog/new beats /blog/[slug], which beats /blog/[...rest] +const better = (a: number[], b: number[]) => { + for (let i = 0; i < Math.max(a.length, b.length); i++) if ((a[i] ?? -1) !== (b[i] ?? -1)) return (a[i] ?? -1) > (b[i] ?? -1); + return false; +}; +const LOCALE = /^\/[a-z]{2}(?:-[A-Za-z]{2,4})?(?=\/|$)/; + +/** Requests per route URL. Each path counts once, for the route Next.js would serve it from. */ +export function routeHits(urls: string[], traffic: Traffic): Map { + const routes = [...new Set(urls)].map((url) => ({ url, ...pattern(url) })); + const out = new Map(routes.map((r) => [r.url, 0])); + const serve = (p: string) => { + let best: (typeof routes)[number] | null = null; + for (const r of routes) if (r.re.test(p) && (!best || better(r.rank, best.rank))) best = r; + return best; + }; + for (const [p, count] of traffic.hits) { + // /fr/about is /about in a Next.js app with i18n routing + const r = serve(p) || (LOCALE.test(p) ? serve(p.replace(LOCALE, '') || '/') : null); + if (r) out.set(r.url, (out.get(r.url) || 0) + count); + } + return out; +} + +const day = (t: number | null) => (t === null ? null : new Date(t).toISOString().slice(0, 10)); + +export function summarize(traffic: Traffic, routes: number, unvisited: number): TrafficSummary { + const days = traffic.from !== null && traffic.to !== null ? Math.max(1, Math.round((traffic.to - traffic.from) / 864e5)) : null; + return { requests: traffic.requests, from: day(traffic.from), to: day(traffic.to), days, routes, unvisited }; +} + +/** "from 2026-09-01 to 2026-10-01 (1,204 requests)" or "in 1,204 requests" when the traffic has no dates. */ +export const span = (t: TrafficSummary) => (t.from && t.to ? `from ${t.from} to ${t.to} (${t.requests.toLocaleString('en-US')} requests)` : `in ${t.requests.toLocaleString('en-US')} requests`); diff --git a/src/types.ts b/src/types.ts index b31cbf6..094b0a2 100644 --- a/src/types.ts +++ b/src/types.ts @@ -56,7 +56,10 @@ export type ScanData = { knipError?: string; since?: string; shortcuts?: Shortcut[]; - zombie: { files: ZombieFile[]; packages: { name: string; file: string; dev: boolean }[]; exports: { file: string; names: string[] }[]; maybe: { file: string; url: string; lines: number }[] }; + /** `maybe`: routes nothing links to; with traffic, routes no request reached (`linked` says whether anything links to them) */ + zombie: { files: ZombieFile[]; packages: { name: string; file: string; dev: boolean }[]; exports: { file: string; names: string[] }[]; maybe: { file: string; url: string; lines: number; linked?: boolean | null }[] }; + /** set when the scan read production traffic (--traffic) */ + traffic?: import('./traffic.ts').TrafficSummary | null; security: { findings: Finding[]; audit: Audit; middlewareAuth: boolean; middleware?: string; inTests: number }; sprawl: { months: Month[]; recent: Month[]; dupes: Clone[] | null; names: { name: string; files: string[] }[]; versions: Version[]; overlaps: Overlap[] }; architecture: { cycles: string[][]; big: { file: string; lines: number; dependents: number }[]; shared: string[]; deep: { file: string; count: number }[]; naming: [string, number][] }; diff --git a/test/e2e.test.ts b/test/e2e.test.ts index 78636a2..a7a2833 100644 --- a/test/e2e.test.ts +++ b/test/e2e.test.ts @@ -26,6 +26,58 @@ const files = { 'components/OldBanner.tsx': 'export function OldBanner() { return
old
}\n', }; +test('zomb --traffic: routes no request reached, with or without links', () => { + const dir = mkdtempSync(path.join(tmpdir(), 'zomb-e2e-traffic-')); + try { + const app = { + 'package.json': JSON.stringify({ name: 'site', private: true, dependencies: { next: '15.0.0', react: '19.0.0' } }), + 'app/page.tsx': 'export default function Home() { return
PricingAboutBlog
; }\n', + 'app/pricing/page.tsx': 'export default function Pricing() { return

pricing

; }\n', + 'app/about/page.tsx': 'export default function About() { return

about

; }\n', + 'app/old-pricing/page.tsx': 'export default function OldPricing() { return

old

; }\n', + 'app/blog/[slug]/page.tsx': 'export default function Post() { return

post

; }\n', + 'app/api/export/route.ts': 'export async function GET() { return Response.json([]); }\n', + }; + for (const [file, body] of Object.entries(app)) { + mkdirSync(path.dirname(path.join(dir, file)), { recursive: true }); + writeFileSync(path.join(dir, file), body); + } + const git = (...args) => execFileSync('git', args, { cwd: dir, stdio: 'ignore' }); + git('init', '-q'); + git('add', '-A'); + git('-c', 'user.name=Dev', '-c', 'user.email=dev@example.com', 'commit', '-q', '-m', 'init'); + const log = path.join(dir, '..', `${path.basename(dir)}.log`); + writeFileSync(log, [ + '1.2.3.4 - - [01/Sep/2026:10:00:00 +0000] "GET /pricing HTTP/1.1" 200 1 "-" "-"', + '1.2.3.4 - - [15/Sep/2026:10:00:00 +0000] "GET /blog/hello-world?ref=x HTTP/1.1" 200 1 "-" "-"', + '{"path":"/api/export","timestamp":"2026-10-01T00:00:00Z"}', + ].join('\n')); + const scan = (...args) => JSON.parse(execFileSync(process.execPath, [cli, dir, '--json', ...args], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] })); + + // the route tasks only: "Is /x still used?" and "Delete /x: ..." + const routeTitles = (tasks) => tasks.map((t) => t.title).filter((t) => /^(Is|Delete) \/[^ ]*(?: still used\?|:)/.test(t)); + const before = routeTitles(scan().tasks); + assert.deepEqual(before.sort(), ['Is /api/export still used?', 'Is /old-pricing still used?'], 'without traffic, unlinked routes are a guess'); + + const { summary, tasks } = scan('--traffic', log); + assert.deepEqual(routeTitles(tasks).sort(), ['Delete /old-pricing: no links and no traffic', 'Is /about still used? No traffic'], 'linked but unvisited is asked about; visited routes, dynamic ones included, are in use'); + const dead = tasks.find((t) => t.title.startsWith('Delete /old-pricing')); + assert.equal(dead.action, 'delete-route'); + assert.equal(dead.safe, false, 'deleting a route changes what users can reach: a person decides'); + assert.match(dead.how, /from 2026-09-01 to 2026-10-01 \(3 requests\)/); + assert.deepEqual(summary.traffic, { requests: 3, from: '2026-09-01', to: '2026-10-01', days: 30, routes: 5, unvisited: 2 }); + assert.equal(scan().summary.traffic, undefined, 'the JSON only changes when traffic is given'); + + const fails = (file) => assert.throws(() => execFileSync(process.execPath, [cli, dir, '--json', '--traffic', file], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }), (e: any) => (assert.match(e.stderr, /--traffic/), true)); + fails(path.join(dir, 'missing.log')); + writeFileSync(log, 'nothing here looks like a request\n'); + fails(log); + rmSync(log); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + test('zomb --json finds every planted mistake', () => { const dir = mkdtempSync(path.join(tmpdir(), 'zomb-e2e-')); try { diff --git a/test/traffic.test.ts b/test/traffic.test.ts new file mode 100644 index 0000000..66a1d82 --- /dev/null +++ b/test/traffic.test.ts @@ -0,0 +1,77 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { gzipSync } from 'node:zlib'; +import { addText, emptyTraffic, normalPath, readLine, readTraffic, routeHits, span, summarize } from '../src/traffic.ts'; + +test('normalPath keeps the path and drops the rest', () => { + assert.equal(normalPath('/blog/post/?utm=x#top'), '/blog/post'); + assert.equal(normalPath('https://site.com/blog//post'), '/blog/post'); + assert.equal(normalPath('/caf%C3%A9'), '/café'); + assert.equal(normalPath('/'), '/'); + assert.equal(normalPath('/100%'), '/100%', 'a stray % is still a path'); + assert.equal(normalPath('blog'), null); + assert.equal(normalPath(42), null); +}); + +test('readLine reads access logs, JSON lines and plain counts, and nothing else', () => { + const nginx = '203.0.113.9 - - [01/Oct/2026:10:00:00 +0200] "GET /pricing?ref=x HTTP/1.1" 200 512 "-" "Mozilla/5.0"'; + assert.deepEqual(readLine(nginx), { path: '/pricing', count: 1, at: Date.parse('2026-10-01T08:00:00Z') }); + // a Vercel log drain entry: the request is in proxy, the time in milliseconds + assert.deepEqual(readLine(JSON.stringify({ source: 'edge', proxy: { method: 'GET', path: '/api/export?x=1', timestamp: 1790000000000 } })), { path: '/api/export', count: 1, at: 1790000000000 }); + assert.deepEqual(readLine('{"path":"/docs","count":12,"timestamp":"2026-09-01"}'), { path: '/docs', count: 12, at: Date.parse('2026-09-01') }); + assert.deepEqual(readLine('{"url":"https://x.dev/a","time":1790000000}'), { path: '/a', count: 1, at: 1790000000000 }, 'seconds become milliseconds'); + assert.deepEqual(readLine('/about 7'), { path: '/about', count: 7, at: null }); + assert.deepEqual(readLine('/about,3'), { path: '/about', count: 3, at: null }); + assert.deepEqual(readLine('/about'), { path: '/about', count: 1, at: null }); + assert.equal(readLine('page,visitors'), null, 'a CSV header is not a request'); + assert.equal(readLine('{"message":"build started"}'), null, 'a log line with no path'); + assert.equal(readLine('{broken'), null); + assert.equal(readLine(''), null); +}); + +test('addText reads a whole JSON array, as Vercel posts it', () => { + const t = emptyTraffic(); + addText(t, JSON.stringify([{ path: '/a', timestamp: 1000 * 86400 * 20000 }, { proxy: { path: '/b' }, timestamp: 1000 * 86400 * 20010 }, { message: 'no path' }])); + assert.deepEqual([...t.hits], [['/a', 1], ['/b', 1]]); + assert.equal(t.requests, 2); + assert.equal(summarize(t, 5, 3).days, 10); +}); + +test('readTraffic streams logs, reads arrays whole, and opens .gz', async () => { + const dir = mkdtempSync(path.join(tmpdir(), 'zomb-traffic-')); + try { + const log = path.join(dir, 'access.log'); + writeFileSync(log, ['1.2.3.4 - - [01/Sep/2026:00:00:00 +0000] "GET /a HTTP/1.1" 200 1', '1.2.3.4 - - [08/Sep/2026:00:00:00 +0000] "POST /a HTTP/2.0" 201 1', 'garbage'].join('\n')); + const arr = path.join(dir, 'drain.json'); + writeFileSync(arr, `\n ${JSON.stringify([{ path: '/b', count: 2 }])}`); + const gz = path.join(dir, 'old.log.gz'); + writeFileSync(gz, gzipSync('/c 5\n/a\n')); + const t = await readTraffic([log, arr, gz]); + assert.deepEqual(Object.fromEntries(t.hits), { '/a': 3, '/b': 2, '/c': 5 }); + const s = summarize(t, 4, 1); + assert.deepEqual(s, { requests: 10, from: '2026-09-01', to: '2026-09-08', days: 7, routes: 4, unvisited: 1 }); + assert.equal(span(s), 'from 2026-09-01 to 2026-09-08 (10 requests)'); + assert.equal(span({ ...s, from: null, to: null }), 'in 10 requests'); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test('routeHits credits each request to the route Next.js would serve it from', () => { + const t = emptyTraffic(); + addText(t, ['/blog/new 2', '/blog/hello 3', '/docs/a/b 1', '/shop 1', '/fr/about 4', '/wp-admin 9', '/api/users/7 1'].join('\n')); + const hits = routeHits(['/blog/[slug]', '/blog/new', '/docs/[...slug]', '/docs', '/shop/[[...slug]]', '/about', '/api/users/[id]', '/unused'], t); + assert.deepEqual(Object.fromEntries(hits), { + '/blog/[slug]': 3, + '/blog/new': 2, + '/docs/[...slug]': 1, + '/docs': 0, + '/shop/[[...slug]]': 1, + '/about': 4, + '/api/users/[id]': 1, + '/unused': 0, + }); +}); From 5184f714159662326d353a54c0b8b60d1dc88801 Mon Sep 17 00:00:00 2001 From: Diip3sh Date: Tue, 13 Oct 2026 12:47:52 +0530 Subject: [PATCH 3/7] Cloud: log drains for traffic, and Copilot without the click Each repo gets a settings page with a drain endpoint and token. Vercel drains are checked by their x-vercel-signature, anything else sends the token as a bearer, gzipped or not. zomb keeps a path and a daily count for 90 days, skips static files and caps a repo at 10,000 paths a day. Once a repo has 14 days of it, push and weekly scans pass the last 30 days to zomb --traffic, so the weekly issue lists the routes nobody requests. "Hand issues to Copilot every week" signs the person in again and keeps that token, sealed with AES-256-GCM (ZOMB_SECRET_KEY, or a key generated into DATA_DIR). The weekly job refreshes it one at a time, since a refresh token works once, and assigns the issue as them when .zomb/config.yml says handoff: copilot. A refused or revoked token is deleted, the issue gets its manual line back, and the dashboard says Copilot stopped. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- cloud/README.md | 22 ++++- cloud/github.ts | 19 ++-- cloud/grants.ts | 37 +++++++ cloud/jobs.ts | 65 +++++++++--- cloud/server.ts | 239 ++++++++++++++++++++++++++++++++++++++++----- cloud/vault.ts | 38 +++++++ src/traffic.ts | 16 +-- test/cloud.test.ts | 147 ++++++++++++++++++++++++++++ 8 files changed, 530 insertions(+), 53 deletions(-) create mode 100644 cloud/grants.ts create mode 100644 cloud/vault.ts diff --git a/cloud/README.md b/cloud/README.md index d73c9f3..478d346 100644 --- a/cloud/README.md +++ b/cloud/README.md @@ -4,7 +4,8 @@ A GitHub App around the zomb CLI. Install it and you're done: no workflow file. - **Every pull request** gets a `zomb` check and one comment, updated on each push. The check fails on new findings at or above `fail-on`. - **Every push to the default branch** is scanned, and the dashboard shows each repo's trend: high security issues, zombie lines, unused packages, architecture and blueprint breaks. -- **Once a week** each repo gets one clean-up issue with zomb's verified task list, kept up to date and closed when the list is empty. It goes to the agent the team already pays for: `@claude` is mentioned in it, or one click in the dashboard assigns it to Copilot (Copilot only takes assignments from a person's token, not an app's). zomb never runs your builds: the agent does the work in your own CI. +- **Once a week** each repo gets one clean-up issue with zomb's verified task list, kept up to date and closed when the list is empty. It goes to the agent the team already pays for: `@claude` is mentioned in it, or it is assigned to Copilot, with one click in the dashboard or automatically every week. zomb never runs your builds: the agent does the work in your own CI. +- **Production traffic**, if you send it, tells each scan which pages and API routes nobody requests any more. ## Run it @@ -32,13 +33,28 @@ Instead of `/setup`, you can pass an existing app's credentials: `GITHUB_APP_ID` ```yaml fail-on: high # the PR check fails on new findings at this level (high, medium, low) weekly: true # the weekly clean-up issue -handoff: claude # claude (mention @claude), copilot (a button in the dashboard) or none +handoff: claude # claude (mention @claude), copilot (a dashboard button, or automatic) or none ``` `.zomb/blueprint.yml` and `.zomb/baseline.json` work as they do in the CLI. +## Traffic: routes nobody visits + +Each repo's settings page (the **settings** link under its name on the dashboard) has a drain endpoint, `ZOMB_URL/drain/`, and a token. + +- **Vercel:** add a Drain for logs with JSON delivery to the endpoint, and use the token as its signature secret. zomb checks the `x-vercel-signature` header. +- **Anything else:** POST an access log, JSON lines or a JSON array with `Authorization: Bearer `, gzipped (`Content-Encoding: gzip`) or not, up to 5 MB at a time. + +zomb keeps only each path and a daily count, for 90 days. Static files are skipped, and a repo keeps at most 10,000 distinct paths a day. Once a repo has 14 days of traffic, push and weekly scans pass the last 30 days to `zomb --traffic`. The weekly issue then lists the routes no request reached. A route that is linked but never visited is asked about, and one with no links and no traffic is marked for deletion. **Make a new token** on the settings page turns the old one off. + +## Copilot, automatically + +Copilot only takes an assignment from a person, never from an app. On a repo's settings page, **Hand issues to Copilot every week** signs you in once more and keeps that GitHub token, so the weekly job can assign the issue as you. It does that only while the repo's `.zomb/config.yml` says `handoff: copilot`, and skips issues Copilot already has. + +The token is encrypted at rest with AES-256-GCM. The key comes from `ZOMB_SECRET_KEY` (32 bytes, as hex or base64), or from a key generated into `DATA_DIR/secret.key`. The generated key sits next to the database, so it protects copies of the database (backups, dumps), not the disk. Set `ZOMB_SECRET_KEY` to keep the two apart. GitHub user tokens last 8 hours, so zomb refreshes them with the refresh token, which GitHub replaces on every use. If GitHub refuses the token, or you revoke zomb in your GitHub settings, zomb deletes it, puts the manual line back on the issue, and the dashboard shows **Copilot stopped** until someone turns it on again. Turning it off on every repo deletes the token. + ## Before you host it for other people Scanning installs a repo's dependencies (with `--ignore-scripts`) and Knip loads its config files, so a scan runs some of that repo's code. Jobs get a clean environment with no server secrets, and the GitHub token never touches the checkout, but they still run as the server's user. That's fine for your own repos. Before scanning code you don't trust, run each job in its own container. The place to do that is `checkout()` in `cloud/jobs.js`. -Not here yet: dead routes from production traffic, billing, and fully automatic Copilot hand-off (it needs a stored user token). +Not here yet: billing. diff --git a/cloud/github.ts b/cloud/github.ts index 97432d0..01cc178 100644 --- a/cloud/github.ts +++ b/cloud/github.ts @@ -36,15 +36,22 @@ export function github({ appId, privateKey, clientId, clientSecret, api = 'https as: (installation: number | string): Api => async (method, url, body) => request(await installationToken(installation), method, url, body), user: (token: string): Api => (method, url, body) => request(token, method, url, body), loginUrl: (state: string, redirect: string) => `${web}/login/oauth/authorize?client_id=${encodeURIComponent(clientId || '')}&state=${state}&redirect_uri=${encodeURIComponent(redirect)}`, - async signIn(code: string) { - const res = await fetch(`${web}/login/oauth/access_token`, { method: 'POST', headers: { accept: 'application/json', 'content-type': 'application/json' }, body: JSON.stringify({ client_id: clientId, client_secret: clientSecret, code }) }); - const data = (await res.json()) as { access_token?: string; error_description?: string }; - if (!data.access_token) throw new Error(data.error_description || 'GitHub sign-in failed'); - return data.access_token; - }, + signIn: (code: string) => userToken({ code }), + // user tokens of a GitHub App expire after 8 hours; the refresh token (6 months) gets a new pair, once + refresh: (refreshToken: string) => userToken({ grant_type: 'refresh_token', refresh_token: refreshToken }), }; + async function userToken(grant: Record): Promise { + const res = await fetch(`${web}/login/oauth/access_token`, { method: 'POST', headers: { accept: 'application/json', 'content-type': 'application/json' }, body: JSON.stringify({ client_id: clientId, client_secret: clientSecret, ...grant }) }); + const data = (await res.json()) as { access_token?: string; expires_in?: number; refresh_token?: string; refresh_token_expires_in?: number; error?: string; error_description?: string }; + if (!data.access_token) throw Object.assign(new Error(data.error_description || 'GitHub sign-in failed'), { code: data.error }); + const at = (s?: number) => (s ? Date.now() + s * 1000 : null); + return { token: data.access_token, expires: at(data.expires_in), refresh: data.refresh_token || null, refreshExpires: at(data.refresh_token_expires_in) }; + } } +/** A user's token. `expires` and the refresh token are null when the app's user tokens never expire. */ +export type UserToken = { token: string; expires: number | null; refresh: string | null; refreshExpires: number | null }; + // Webhooks are signed with the app's secret: anything else is someone pretending to be GitHub. export function verify(secret: string, body: Buffer | string, signature: string | string[] | undefined = '') { const want = Buffer.from(`sha256=${createHmac('sha256', secret).update(body).digest('hex')}`); diff --git a/cloud/grants.ts b/cloud/grants.ts new file mode 100644 index 0000000..a94e8cf --- /dev/null +++ b/cloud/grants.ts @@ -0,0 +1,37 @@ +// The GitHub tokens people hand zomb so it can assign the weekly issue to Copilot without a click: +// Copilot only takes assignments from a person, never from an app. Sealed at rest, refreshed one at a time. +import type { UserToken } from './github.ts'; +import type { Vault } from './vault.ts'; + +const EARLY = 5 * 60_000; + +export function grants({ db, vault, gh }: { db: any; vault: Vault; gh: { refresh: (t: string) => Promise } }) { + db.exec('CREATE TABLE IF NOT EXISTS grants (login TEXT PRIMARY KEY, token TEXT NOT NULL, expires INTEGER, refresh TEXT, refresh_expires INTEGER)'); + const put = (login: string, t: UserToken) => + db + .prepare('INSERT INTO grants (login, token, expires, refresh, refresh_expires) VALUES (?, ?, ?, ?, ?) ON CONFLICT(login) DO UPDATE SET token = excluded.token, expires = excluded.expires, refresh = excluded.refresh, refresh_expires = excluded.refresh_expires') + .run(login, vault.seal(t.token), t.expires, t.refresh && vault.seal(t.refresh), t.refreshExpires); + // a refresh token works once: two jobs refreshing the same grant at the same time would lose it + const inflight = new Map>(); + + async function token(login: string): Promise { + const g = db.prepare('SELECT * FROM grants WHERE login = ?').get(login) as any; + if (!g) throw Object.assign(new Error(`no grant for ${login}`), { code: 'no_grant' }); + if (g.expires === null || g.expires - Date.now() > EARLY) return vault.open(g.token); + if (!g.refresh || (g.refresh_expires && g.refresh_expires < Date.now())) throw Object.assign(new Error(`${login}'s grant expired`), { code: 'bad_refresh_token' }); + const fresh = await gh.refresh(vault.open(g.refresh)); + put(login, fresh); + return fresh.token; + } + + return { + save: put, + has: (login: string) => Boolean(login && db.prepare('SELECT 1 FROM grants WHERE login = ?').get(login)), + drop: (login: string) => db.prepare('DELETE FROM grants WHERE login = ?').run(login), + token(login: string) { + if (!inflight.has(login)) inflight.set(login, token(login).finally(() => inflight.delete(login))); + return inflight.get(login)!; + }, + }; +} +export type Grants = ReturnType; diff --git a/cloud/jobs.ts b/cloud/jobs.ts index c3ddd83..f60dfbe 100644 --- a/cloud/jobs.ts +++ b/cloud/jobs.ts @@ -1,6 +1,6 @@ // The work behind each GitHub event: check the code out, run zomb on it, and report back on GitHub. import { execFile } from 'node:child_process'; -import { existsSync, readFileSync } from 'node:fs'; +import { existsSync, readFileSync, writeFileSync } from 'node:fs'; import { mkdir, mkdtemp, realpath, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import path from 'node:path'; @@ -9,10 +9,19 @@ import { parse } from 'yaml'; import { toMarkdown } from '../src/tasks.ts'; import { packageManager } from '../src/fix.ts'; import type { Api } from './github.ts'; +import type { Grants } from './grants.ts'; import type { Task } from '../src/types.ts'; type Result = { summary: Record; failing: number; tasks: Task[]; baseline?: boolean }; +/** Traffic younger than this says too little: a page visited once a month would look dead. */ +export const TRAFFIC_MIN_DAYS = 14; +const COPILOT = /^copilot(-swe-agent)?(\[bot\])?$/i; + +// Copilot takes the issue only from a person's token; the app's own token is refused. +export const assignCopilot = (user: Api, repo: string, issue: number) => + user('POST', `/repos/${repo}/issues/${issue}/assignees`, { assignees: ['copilot-swe-agent[bot]'], agent_assignment: { target_repo: repo } }); + const CLI = path.join(import.meta.dirname, '..', 'src', 'cli.ts'); const exec = promisify(execFile); const MARK = ''; @@ -39,7 +48,7 @@ async function checkout(cloneUrl: string, token: string, refspecs: string[], ref if (['package-lock.json', 'pnpm-lock.yaml', 'yarn.lock', 'bun.lock', 'bun.lockb'].some((f) => existsSync(path.join(dir, f)))) await run(pm, [...INSTALL[pm], '--ignore-scripts']).catch(() => {}); const file = path.join(dir, '.zomb', 'config.yml'); const config = { 'fail-on': 'high', weekly: true, handoff: 'none', ...(existsSync(file) ? parse(readFileSync(file, 'utf8')) : {}) }; - return { dir, config, run, cleanup: () => rm(root, { recursive: true, force: true }) }; + return { root, dir, config, run, cleanup: () => rm(root, { recursive: true, force: true }) }; } catch (e) { await rm(root, { recursive: true, force: true }); // execFile puts the command in its message, and the fetch URL carries the token: it would reach the log and the check run @@ -56,10 +65,36 @@ async function scan(co: { dir: string; run: (cmd: string, args: string[], timeou return JSON.parse(out); } -export function jobs({ gh, db, appId, log = console.error }: { gh: any; db: any; appId?: string | number; log?: (s: string) => void }) { +export function jobs({ gh, db, appId, log = console.error, grants }: { gh: any; db: any; appId?: string | number; log?: (s: string) => void; grants?: Grants }) { const record = (repo: number | string, sha: string | null, kind: string, result: Result) => db.prepare('INSERT INTO scans (repo, sha, kind, at, summary, failing, tasks) VALUES (?, ?, ?, ?, ?, ?, ?)').run(repo, sha, kind, new Date().toISOString(), JSON.stringify(result.summary), result.failing, result.tasks.length); + // The last 30 days of the repo's log drain, as a file next to the checkout for --traffic. None until it covers 14 days. + function traffic(repo: number | string, root: string): string[] { + const first = (db.prepare('SELECT min(day) AS day FROM traffic WHERE repo = ? AND hits > 0').get(repo) as any)?.day; + if (!first || Date.parse(first) > Date.now() - TRAFFIC_MIN_DAYS * 864e5) return []; + const rows = db.prepare("SELECT path, day, hits FROM traffic WHERE repo = ? AND hits > 0 AND day >= date('now', '-30 days')").all(repo) as any[]; + if (!rows.length) return []; + const file = path.join(root, 'traffic.ndjson'); + writeFileSync(file, `${rows.map((r) => JSON.stringify({ path: r.path, count: r.hits, timestamp: r.day })).join('\n')}\n`); + return ['--traffic', file]; + } + + // Assign the weekly issue to Copilot with the grant someone turned on in the repo's settings. + async function handOff(repo: any, issue: any) { + if (issue.assignees?.some((a: any) => COPILOT.test(a.login))) return; + try { + await assignCopilot(gh.user(await grants!.token(repo.copilot_login)), repo.name, issue.number); + } catch (e) { + const err = e as { code?: string; status?: number; message: string }; + // a refused or revoked token stays refused: drop it and ask for a new sign-in + const dead = err.code === 'bad_refresh_token' || err.code === 'no_grant' || err.status === 401; + if (dead) grants!.drop(repo.copilot_login); + db.prepare('UPDATE repos SET handoff_error = ? WHERE id = ?').run(dead ? `GitHub no longer accepts ${repo.copilot_login}'s sign-in` : `GitHub refused the assignment (${err.message.slice(0, 160)})`, repo.id); + throw e; + } + } + async function pullRequest({ repository: r, pull_request: pr, installation }: any) { const api = gh.as(installation.id); const check = await api('POST', `/repos/${r.full_name}/check-runs`, { name: 'zomb', head_sha: pr.head.sha, status: 'in_progress' }); @@ -89,7 +124,7 @@ export function jobs({ gh, db, appId, log = console.error }: { gh: any; db: any; if (ref !== `refs/heads/${r.default_branch}` || /^0+$/.test(after)) return; const co = await checkout(r.clone_url, await gh.installationToken(installation.id), [`+${ref}:refs/remotes/origin/${r.default_branch}`], `origin/${r.default_branch}`); try { - record(r.id, after, 'push', await scan(co, [])); + record(r.id, after, 'push', await scan(co, traffic(r.id, co.root))); } finally { await co.cleanup(); } @@ -100,7 +135,7 @@ export function jobs({ gh, db, appId, log = console.error }: { gh: any; db: any; const info = await gh.as(repo.installation)('GET', `/repos/${repo.name}`); const co = await checkout(info.clone_url, await gh.installationToken(repo.installation), [`+refs/heads/${info.default_branch}:refs/remotes/origin/${info.default_branch}`], `origin/${info.default_branch}`); try { - record(repo.id, null, 'install', await scan(co, [])); + record(repo.id, null, 'install', await scan(co, traffic(repo.id, co.root))); } catch (e) { // HTML or Python only: nothing for zomb to read, so remember that instead of retrying on every start if (!/no JS\/TS files/.test((e as Error).message)) throw e; @@ -119,16 +154,24 @@ export function jobs({ gh, db, appId, log = console.error }: { gh: any; db: any; try { db.prepare('UPDATE repos SET weekly_at = ? WHERE id = ?').run(new Date().toISOString(), repo.id); if (co.config.weekly === false) return null; - const result = await scan(co, []); + const result = await scan(co, traffic(repo.id, co.root)); record(repo.id, null, 'weekly', result); const open = (await api('GET', `/repos/${repo.name}/issues?state=open&per_page=100`)).find((i: any) => i.body?.startsWith(WEEKLY) && !i.pull_request); if (!result.tasks.length) { if (open) await api('PATCH', `/repos/${repo.name}/issues/${open.number}`, { state: 'closed', body: `${WEEKLY}\nNothing left to clean up. 🎉` }); return null; } - const body = weeklyBody(result, co.config.handoff, !open); - if (open) return (await api('PATCH', `/repos/${repo.name}/issues/${open.number}`, { body })).number; - return (await api('POST', `/repos/${repo.name}/issues`, { title: 'zomb: weekly clean-up', body })).number; + const auto = co.config.handoff === 'copilot' && repo.copilot_login && !repo.handoff_error && grants?.has(repo.copilot_login) ? repo.copilot_login : null; + const body = weeklyBody(result, co.config.handoff, !open, auto); + const issue = open ? await api('PATCH', `/repos/${repo.name}/issues/${open.number}`, { body }) : await api('POST', `/repos/${repo.name}/issues`, { title: 'zomb: weekly clean-up', body }); + if (auto) { + await handOff(repo, issue).catch(async (e) => { + log(`${repo.name} weekly: Copilot hand-off failed: ${e.message}`); + // the issue said it was handed over; say how to hand it over instead + await api('PATCH', `/repos/${repo.name}/issues/${issue.number}`, { body: weeklyBody(result, co.config.handoff, !open, null) }); + }); + } + return issue.number; } finally { await co.cleanup(); } @@ -138,7 +181,7 @@ export function jobs({ gh, db, appId, log = console.error }: { gh: any; db: any; } // The issue an agent (or a person) works from: the safe clean-up first, then what needs a decision. -function weeklyBody(result: Result, handoff: string, first: boolean) { +function weeklyBody(result: Result, handoff: string, first: boolean, auto: string | null) { const s = result.summary; const line = (t: Task) => `- [ ] **${t.title}** \n \`${t.where.replace(/`/g, "'")}\`: ${t.how}`; const safe = result.tasks.filter((t) => t.safe); @@ -150,7 +193,7 @@ function weeklyBody(result: Result, handoff: string, first: boolean) { rest.length && `### Needs a look (${rest.length})\n${rest.slice(0, 40).map(line).join('\n')}`, `### How to work on this\nFollow [the zomb clean-up playbook](https://github.com/Open-Dev-Society/zomb/blob/main/skills/zomb-clean/SKILL.md): security first, then \`npx zomb fix\`, then one reviewed change at a time. Never print a secret; list keys that need rotating for a human.`, first && handoff === 'claude' && '@claude please work through this list, following the playbook above, and open a pull request.', - handoff === 'copilot' && '_Assign this issue to Copilot from the zomb dashboard to hand it over._', + handoff === 'copilot' && (auto ? `_zomb assigns this issue to Copilot every week, on behalf of ${auto}._` : '_Assign this issue to Copilot from the zomb dashboard to hand it over._'), ]; return parts.filter(Boolean).join('\n\n'); } diff --git a/cloud/server.ts b/cloud/server.ts index e797d16..565495d 100644 --- a/cloud/server.ts +++ b/cloud/server.ts @@ -2,17 +2,27 @@ // zomb Cloud: a GitHub App. Install it and every pull request gets a zomb check and comment (no workflow file), // every push to the default branch feeds the trends dashboard, and each repo gets a weekly clean-up issue // handed to the agent the team already uses. Self-host it, or use ours. -import { randomBytes } from 'node:crypto'; +import { createHmac, randomBytes, timingSafeEqual } from 'node:crypto'; import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; import { createServer } from 'node:http'; import { createRequire } from 'node:module'; import path from 'node:path'; import { DatabaseSync } from 'node:sqlite'; +import { gunzipSync } from 'node:zlib'; +import { hitsOf } from '../src/traffic.ts'; import { github, verify } from './github.ts'; -import { jobs } from './jobs.ts'; +import { grants } from './grants.ts'; +import { assignCopilot, jobs, TRAFFIC_MIN_DAYS } from './jobs.ts'; +import { vault } from './vault.ts'; type Session = { id: string; login: string; token: string; installations: number[]; expires: number }; +const KEEP_DAYS = 90; +const MAX_BODY = 5 * 1024 * 1024; +const MAX_PATHS = 10_000; // distinct paths per repo per day +// files, not routes: a scan never asks about them, so they are not worth a row +const STATIC = /^\/(_next|_vercel|static|assets)\/|\/[^/]+\.[a-z0-9]{1,8}$/i; + function config(env: Record = process.env) { const data = env.DATA_DIR || path.join(import.meta.dirname, 'data'); // written by /setup when the app was created from its manifest; env vars win @@ -39,12 +49,19 @@ export function start(cfg: ReturnType = config(), { log = console db.exec(`CREATE TABLE IF NOT EXISTS repos (id INTEGER PRIMARY KEY, installation INTEGER NOT NULL, name TEXT NOT NULL, weekly_at TEXT); CREATE TABLE IF NOT EXISTS scans (id INTEGER PRIMARY KEY AUTOINCREMENT, repo INTEGER NOT NULL, sha TEXT, kind TEXT, at TEXT, summary TEXT, failing INTEGER, tasks INTEGER); CREATE TABLE IF NOT EXISTS sessions (id TEXT PRIMARY KEY, login TEXT, token TEXT, installations TEXT, expires INTEGER);`); - // why a repo has no scan, when it never can have one (no JS/TS); added after the first release - if (!db.prepare("SELECT 1 FROM pragma_table_info('repos') WHERE name = 'note'").get()) db.exec('ALTER TABLE repos ADD COLUMN note TEXT'); + // production traffic from log drains: a path and a daily count, nothing about who asked + db.exec('CREATE TABLE IF NOT EXISTS traffic (repo INTEGER NOT NULL, day TEXT NOT NULL, path TEXT NOT NULL, hits INTEGER NOT NULL, PRIMARY KEY (repo, day, path))'); + // added after the first release. note: why a repo has no scan, when it never can have one (no JS/TS); + // drain_secret: the sealed token its log drain signs with; copilot_login: whose grant hands its weekly issue to Copilot + for (const col of ['note', 'drain_secret', 'copilot_login', 'handoff_error']) + if (!db.prepare("SELECT 1 FROM pragma_table_info('repos') WHERE name = ?").get(col)) db.exec(`ALTER TABLE repos ADD COLUMN ${col} TEXT`); // built either way: they only make closures, and `ready` gates every call that needs real credentials const ready = Boolean(cfg.appId && cfg.privateKey && cfg.webhookSecret); const gh = github(cfg); - const work = jobs({ gh, db, appId: cfg.appId, log }); + const secrets = vault(cfg.data); + const tokens = grants({ db, vault: secrets, gh }); + const work = jobs({ gh, db, appId: cfg.appId, log, grants: tokens }); + const forget = (repo: number) => db.prepare('DELETE FROM traffic WHERE repo = ?').run(repo); // ponytail: in-memory queue, two jobs at a time; a restart drops queued jobs (the next push or week redoes them) const queue: [string, () => Promise][] = []; @@ -67,21 +84,28 @@ export function start(cfg: ReturnType = config(), { log = console } }; function onEvent(event: string | string[] | undefined, p: any) { - if (event === 'installation' && p.action === 'deleted') db.prepare('DELETE FROM repos WHERE installation = ?').run(p.installation.id); - else if (event === 'installation' && p.repositories) addRepos(p.installation.id, p.repositories); + if (event === 'installation' && p.action === 'deleted') { + for (const r of db.prepare('SELECT id FROM repos WHERE installation = ?').all(p.installation.id) as any[]) forget(r.id); + db.prepare('DELETE FROM repos WHERE installation = ?').run(p.installation.id); + } else if (event === 'installation' && p.repositories) addRepos(p.installation.id, p.repositories); else if (event === 'installation_repositories') { addRepos(p.installation.id, p.repositories_added || []); - for (const r of p.repositories_removed || []) db.prepare('DELETE FROM repos WHERE id = ?').run(r.id); + for (const r of p.repositories_removed || []) forget(r.id), db.prepare('DELETE FROM repos WHERE id = ?').run(r.id); + } else if (event === 'github_app_authorization' && p.action === 'revoked') { + // someone revoked zomb on GitHub: their token is dead, so stop handing issues over in their name + tokens.drop(p.sender.login); + db.prepare('UPDATE repos SET handoff_error = ? WHERE copilot_login = ?').run(`${p.sender.login} revoked zomb's access`, p.sender.login); } else if (event === 'pull_request' && ['opened', 'synchronize', 'reopened', 'ready_for_review'].includes(p.action)) enqueue(`${p.repository.full_name}#${p.number}`, () => work.pullRequest(p)); else if (event === 'push') enqueue(`${p.repository.full_name} push`, () => work.push(p)); } - // every hour: repos whose weekly clean-up is due - const weeklyDue = () => { + // every hour: repos whose weekly clean-up is due, and traffic older than anything a scan reads + const hourly = () => { const due = db.prepare("SELECT * FROM repos WHERE weekly_at IS NULL OR datetime(weekly_at) < datetime('now', '-7 days')").all() as any[]; for (const repo of due) enqueue(`${repo.name} weekly`, () => work.weekly(repo)); + db.prepare(`DELETE FROM traffic WHERE day < date('now', '-${KEEP_DAYS} days')`).run(); }; - const timer = ready ? setInterval(weeklyDue, 3600_000) : undefined; + const timer = ready ? setInterval(hourly, 3600_000) : undefined; // repos installed before their first scan ran (or whose first scan a restart dropped) catch up on start // ponytail: a repo that fails for another reason (empty, no default branch) is retried on every start if (ready) for (const repo of db.prepare('SELECT * FROM repos WHERE note IS NULL AND id NOT IN (SELECT repo FROM scans)').all() as any[]) enqueue(`${repo.name} first scan`, () => work.first(repo)); @@ -94,7 +118,81 @@ export function start(cfg: ReturnType = config(), { log = console return s ? { ...s, installations: JSON.parse(s.installations) } : null; }; - const send = (res: any, status: number, body: string, headers: Record = {}) => (res.writeHead(status, { 'content-type': 'text/html; charset=utf-8', ...headers }), res.end(body)); + const send = (res: any, status: number, body: string, headers: Record = {}) => (res.writeHead(status, { 'content-type': 'text/html; charset=utf-8', ...headers }), res.end(body)); + const json = (res: any, status: number, data: unknown) => send(res, status, JSON.stringify(data), { 'content-type': 'application/json' }); + + // ---------- traffic: a log drain posts requests, zomb keeps a path and a daily count per repo + const drainSecret = (repo: any): string => { + if (repo.drain_secret) return secrets.open(repo.drain_secret); + const secret = randomBytes(24).toString('base64url'); + db.prepare('UPDATE repos SET drain_secret = ? WHERE id = ?').run(secrets.seal(secret), repo.id); + return secret; + }; + const same = (a: string, b: string) => a.length === b.length && timingSafeEqual(Buffer.from(a), Buffer.from(b)); + function drain(repo: any, raw: Buffer, headers: Record) { + if (!repo?.drain_secret) return { status: 404, error: 'no drain for this repo: open its settings in zomb first' }; + const secret = secrets.open(repo.drain_secret); + let text = raw; + try { + if (/gzip/i.test(headers['content-encoding'] || '')) text = gunzipSync(raw); + } catch { + return { status: 400, error: 'bad gzip body' }; + } + // Vercel signs the body with HMAC-SHA1; anything else sends the token as a bearer + const bearer = String(headers.authorization || '').replace(/^Bearer\s+/i, ''); + const signature = String(headers['x-vercel-signature'] || ''); + const signed = (b: Buffer) => createHmac('sha1', secret).update(b).digest('hex'); + if (!(bearer && same(bearer, secret)) && !(signature && (same(signature, signed(raw)) || same(signature, signed(text))))) return { status: 401, error: 'bad token or signature' }; + const today = new Date().toISOString().slice(0, 10); + const oldest = new Date(Date.now() - KEEP_DAYS * 864e5).toISOString().slice(0, 10); + const tally = new Map(); + let requests = 0; + for (const h of hitsOf(text.toString('utf8'))) { + if (!h.count || STATIC.test(h.path) || h.path.length > 512) continue; + const at = h.at !== null && Number.isFinite(h.at) && Math.abs(h.at) < 8.64e15 ? new Date(h.at).toISOString().slice(0, 10) : today; + const day = at > today ? today : at; + if (day < oldest) continue; + tally.set(`${day}\n${h.path}`, (tally.get(`${day}\n${h.path}`) || 0) + h.count); + requests += h.count; + } + const bump = db.prepare('UPDATE traffic SET hits = hits + ? WHERE repo = ? AND day = ? AND path = ?'); + const insert = db.prepare('INSERT INTO traffic (repo, day, path, hits) VALUES (?, ?, ?, ?)'); + const count = db.prepare('SELECT count(*) AS n FROM traffic WHERE repo = ? AND day = ?'); + const paths = new Map(); + let dropped = 0; + db.exec('BEGIN'); + try { + for (const [key, hits] of tally) { + const [day, p] = key.split('\n'); + if (bump.run(hits, repo.id, day, p).changes) continue; + // a site with endless distinct URLs (search pages, ids in paths) must not fill the disk + const n = paths.get(day) ?? (count.get(repo.id, day) as any).n; + if (n >= MAX_PATHS) { + dropped++; + continue; + } + insert.run(repo.id, day, p, hits); + paths.set(day, n + 1); + } + db.exec('COMMIT'); + } catch (e) { + db.exec('ROLLBACK'); + throw e; + } + return { status: 200, requests, dropped }; + } + + const bodyOf = async (req: AsyncIterable, limit: number): Promise => { + const chunks: Buffer[] = []; + let size = 0; + for await (const c of req) { + size += c.length; + if (size > limit) return null; + chunks.push(c); + } + return Buffer.concat(chunks); + }; + const server = createServer(async (req, res) => { const url = new URL(req.url || '/', cfg.url); try { @@ -107,6 +205,13 @@ export function start(cfg: ReturnType = config(), { log = console return onEvent(req.headers['x-github-event'], JSON.parse(body.toString())); } if (url.pathname === '/health') return send(res, 200, 'ok', { 'content-type': 'text/plain' }); + const drainFor = url.pathname.match(/^\/drain\/(\d+)$/); + if (drainFor && req.method === 'POST') { + const raw = await bodyOf(req, MAX_BODY); + if (!raw) return json(res, 413, { error: `bodies are capped at ${MAX_BODY / 1024 / 1024} MB: send smaller batches` }); + const { status, ...out } = drain(db.prepare('SELECT * FROM repos WHERE id = ?').get(Number(drainFor[1])), raw, req.headers); + return json(res, status, out); + } if (url.pathname === '/setup') return send(res, 200, setupPage(cfg, ready)); if (url.pathname === '/setup/done') { if (ready) return send(res, 400, 'Already set up.'); @@ -122,13 +227,21 @@ export function start(cfg: ReturnType = config(), { log = console } if (url.pathname === '/callback') { if (!url.searchParams.get('state') || url.searchParams.get('state') !== cookies(req).zomb_state) return send(res, 400, 'Sign-in expired. Try again.'); - const token = await gh.signIn(url.searchParams.get('code') || ''); - const user = gh.user(token); + const signedIn = await gh.signIn(url.searchParams.get('code') || ''); + const user = gh.user(signedIn.token); const me = await user('GET', '/user'); const { installations } = await user('GET', '/user/installations?per_page=100'); const id = randomBytes(24).toString('hex'); - db.prepare('INSERT INTO sessions (id, login, token, installations, expires) VALUES (?, ?, ?, ?, ?)').run(id, me.login, token, JSON.stringify(installations.map((i) => i.id)), Date.now() + 8 * 3600_000); - return send(res, 302, '', { location: '/', 'set-cookie': `zomb=${id}; HttpOnly; SameSite=Lax; Path=/; Max-Age=28800${secure}` }); + db.prepare('INSERT INTO sessions (id, login, token, installations, expires) VALUES (?, ?, ?, ?, ?)').run(id, me.login, signedIn.token, JSON.stringify(installations.map((i) => i.id)), Date.now() + 8 * 3600_000); + const cookie = `zomb=${id}; HttpOnly; SameSite=Lax; Path=/; Max-Age=28800${secure}`; + // signing in to turn on automatic hand-off: keep this token, sealed, for the weekly job + const grantFor = db.prepare('SELECT * FROM repos WHERE id = ?').get(Number(cookies(req).zomb_grant || 0)) as any; + if (grantFor && installations.some((i) => i.id === grantFor.installation)) { + tokens.save(me.login, signedIn); + db.prepare('UPDATE repos SET copilot_login = ?, handoff_error = NULL WHERE id = ?').run(me.login, grantFor.id); + return send(res, 302, '', { location: `/repos/${grantFor.id}`, 'set-cookie': [cookie, 'zomb_grant=; Path=/; Max-Age=0'] }); + } + return send(res, 302, '', { location: '/', 'set-cookie': cookie }); } if (url.pathname === '/logout') { db.prepare('DELETE FROM sessions WHERE id = ?').run(cookies(req).zomb || ''); @@ -136,15 +249,41 @@ export function start(cfg: ReturnType = config(), { log = console } const me = session(req); if (url.pathname === '/') return send(res, 200, me ? dashboard(db, me, cfg, url.searchParams.get('view') || 'all') : landing(cfg)); - // hand the weekly issue to Copilot: Copilot only takes assignments from a person's token, so this is a click, not a cron - const copilot = url.pathname.match(/^\/repos\/(\d+)\/copilot$/); - if (copilot && req.method === 'POST' && me && req.headers.origin === new URL(cfg.url).origin) { - const repo = db.prepare('SELECT * FROM repos WHERE id = ?').get(Number(copilot[1])) as any; - if (!repo || !me.installations.includes(repo.installation)) return send(res, 404, 'Not found'); - const issue = (await gh.as(repo.installation)('GET', `/repos/${repo.name}/issues?state=open&per_page=100`)).find((i: any) => i.body?.startsWith('')); - if (!issue) return send(res, 404, page('No issue', '

No open weekly clean-up issue yet. It is created on the next weekly run.

')); - await gh.user(me.token)('POST', `/repos/${repo.name}/issues/${issue.number}/assignees`, { assignees: ['copilot-swe-agent[bot]'], agent_assignment: { target_repo: repo.name } }); - return send(res, 302, '', { location: '/' }); + + // ---------- one repo: its settings, and the forms on them + const at = url.pathname.match(/^\/repos\/(\d+)(\/[a-z/]+)?$/); + if (at) { + if (!me) return send(res, 302, '', { location: '/login' }); + const repo = db.prepare('SELECT * FROM repos WHERE id = ?').get(Number(at[1])) as any; + if (!repo || !me.installations.includes(repo.installation)) return send(res, 404, page('Not found', '

Not found.

')); + const action = at[2] || ''; + if (req.method === 'GET' && !action) return send(res, 200, settingsPage(cfg, repo, drainSecret(repo), trafficStats(db, repo.id), tokens.has(repo.copilot_login))); + // forms only from our own pages + if (req.method !== 'POST' || req.headers.origin !== new URL(cfg.url).origin) return send(res, 404, page('Not found', '

Not found.

')); + const back = { location: `/repos/${repo.id}` }; + if (action === '/drain') { + db.prepare('UPDATE repos SET drain_secret = NULL WHERE id = ?').run(repo.id); + return send(res, 302, '', back); + } + if (action === '/copilot/auto') { + const form = new URLSearchParams(String((await bodyOf(req, 4096)) || '')); + if (form.get('on') === '1') { + // sign in again so the token zomb keeps is one the person handed over for this, not a session's + const state = randomBytes(16).toString('hex'); + const short = `HttpOnly; SameSite=Lax; Path=/; Max-Age=600${secure}`; + return send(res, 302, '', { location: gh.loginUrl(state, `${cfg.url}/callback`), 'set-cookie': [`zomb_state=${state}; ${short}`, `zomb_grant=${repo.id}; ${short}`] }); + } + db.prepare('UPDATE repos SET copilot_login = NULL, handoff_error = NULL WHERE id = ?').run(repo.id); + if (repo.copilot_login && !db.prepare('SELECT 1 FROM repos WHERE copilot_login = ?').get(repo.copilot_login)) tokens.drop(repo.copilot_login); + return send(res, 302, '', back); + } + // hand this week's issue to Copilot now, as the person clicking + if (action === '/copilot') { + const issue = (await gh.as(repo.installation)('GET', `/repos/${repo.name}/issues?state=open&per_page=100`)).find((i: any) => i.body?.startsWith('')); + if (!issue) return send(res, 404, page('No issue', '

No open weekly clean-up issue yet. It is created on the next weekly run.

')); + await assignCopilot(gh.user(me.token), repo.name, issue.number); + return send(res, 302, '', { location: '/' }); + } } send(res, 404, page('Not found', '

Not found.

')); } catch (e) { @@ -318,6 +457,9 @@ td .warn{color:var(--warn);font-weight:600} /* one quiet action per row: 24 bordered buttons shout louder than the data */ .handoff{background:none;border:0;padding:0;font:500 13px/1 var(--sans);color:var(--mute);cursor:pointer;white-space:nowrap;text-decoration:underline;text-underline-offset:3px;text-decoration-color:var(--line-2);transition:color .15s var(--ease)} .handoff:hover{color:var(--ink);text-decoration-color:currentColor} +.auto{font:500 13px/1 var(--sans);color:var(--accent-ink);white-space:nowrap} +.when a{text-decoration:underline;text-underline-offset:2px;text-decoration-color:var(--line-2)} +.when a:hover{color:var(--ink)} .trend{display:inline-block;margin-left:9px;vertical-align:middle} .act{display:flex;justify-content:flex-end} @@ -428,9 +570,15 @@ ${fig(deletes === null ? '—' : num(deletes), 'Deleted per 100 added', deletes const tick = !r.last ? 'transparent' : r.last.security.high ? 'var(--high)' : r.last.blueprint ? 'var(--warn)' : r.last.zombie.lines ? 'var(--line-2)' : 'var(--accent)'; return ` ${esc(r.name)} -${r.at ? `scanned ${esc(String(r.at).slice(0, 10))}` : r.note ? 'not scanned' : 'not scanned yet'}${r.weekly_at ? ` · cleaned ${esc(String(r.weekly_at).slice(0, 10))}` : ''} +${r.at ? `scanned ${esc(String(r.at).slice(0, 10))}` : r.note ? 'not scanned' : 'not scanned yet'}${r.weekly_at ? ` · cleaned ${esc(String(r.weekly_at).slice(0, 10))}` : ''} · settings ${!r.last ? `${r.note ? `Nothing to scan: ${esc(r.note)}` : 'Not scanned yet'}` : `${cell(r, (s) => s.security.high, { trend: true, tone: 'hit' })}${cell(r, (s) => s.zombie.lines, { trend: true })}${cell(r, (s) => s.zombie.packages)}${cell(r, (s) => s.architecture.cycles + s.architecture.bigFiles)}${cell(r, (s) => s.blueprint || 0, { tone: 'warn' })}`} -
`; +
${ + r.handoff_error + ? `Copilot stopped` + : r.copilot_login + ? 'Copilot, automatic' + : `
` + }
`; }) .join(''); @@ -460,6 +608,43 @@ ${!r.last ? `${r.note ? `Nothing to scan: ${esc( `; } +function trafficStats(db: any, repo: number) { + return db.prepare('SELECT min(day) AS first, count(DISTINCT day) AS days, coalesce(sum(hits), 0) AS hits FROM traffic WHERE repo = ?').get(repo) as { first: string | null; days: number; hits: number }; +} + +// One repo's settings: where its log drain posts, and whether its weekly issue goes to Copilot by itself. +function settingsPage(cfg: ReturnType, repo: any, secret: string, stats: ReturnType, granted: boolean) { + const endpoint = `${cfg.url}/drain/${repo.id}`; + const enough = stats.first !== null && Date.parse(stats.first) <= Date.now() - TRAFFIC_MIN_DAYS * 864e5; + const status = stats.hits + ? `${num(stats.hits)} requests over ${num(stats.days)} day${stats.days === 1 ? '' : 's'}, since ${esc(stats.first)}. ${enough ? 'Scans use the last 30 days.' : `Scans start using it once it covers ${TRAFFIC_MIN_DAYS} days.`}` + : 'Nothing received yet.'; + const form = (on: boolean, label: string, tone: string) => `
`; + const copilot = + repo.copilot_login && granted && !repo.handoff_error + ? `

Each weekly clean-up issue is assigned to Copilot automatically, as ${esc(repo.copilot_login)}, while the repo's .zomb/config.yml says handoff: copilot.

${form(false, 'Turn off', 'btn-g')}` + : `${repo.handoff_error ? `

Stopped: ${esc(repo.handoff_error)}. Turn it on again to resume.

` : ''}

Copilot only takes an assignment from a person. Turn this on and zomb keeps your GitHub sign-in, encrypted, to assign each weekly clean-up issue to Copilot for you. It needs handoff: copilot in the repo's .zomb/config.yml. Turn it off on every repo and zomb deletes the sign-in.

${form(true, 'Hand issues to Copilot every week', 'btn-p')}`; + return page( + repo.name, + ` +

repo settings

${esc(repo.name)}

+

Production traffic

+

Send zomb your production requests and each scan lists the routes nobody visits. It keeps only each path and a daily count, for ${KEEP_DAYS} days.

+

${status}

+
Endpoint
${esc(endpoint)}
Token
${esc(secret)}
+

Vercel: add a Drain for logs with JSON delivery to the endpoint, and use the token as its signature secret.

+

Anything else: POST an access log, JSON lines or a JSON array, up to ${MAX_BODY / 1024 / 1024} MB at a time, with the token as a bearer:

+
curl -X POST ${esc(endpoint)} -H "Authorization: Bearer ${esc(secret)}" --data-binary @access.log
+
+

Copilot

+${copilot} +

Back to the dashboard

`, + ); +} + // Create the GitHub App from a manifest: one click on GitHub, and the credentials come back here. function setupPage(cfg: ReturnType, ready: boolean) { if (ready) return page('Set up', '

Already set up

Go to the dashboard

'); diff --git a/cloud/vault.ts b/cloud/vault.ts new file mode 100644 index 0000000..dd04971 --- /dev/null +++ b/cloud/vault.ts @@ -0,0 +1,38 @@ +// Secrets at rest: drain tokens and the GitHub tokens people hand zomb for automatic Copilot hand-off. +// AES-256-GCM with a key from ZOMB_SECRET_KEY, or one generated into DATA_DIR/secret.key on first start. +// The key file sits next to the database, so this guards copies of the database (backups, dumps), not the disk. +import { createCipheriv, createDecipheriv, randomBytes } from 'node:crypto'; +import { existsSync, readFileSync, writeFileSync } from 'node:fs'; +import path from 'node:path'; + +export type Vault = { seal: (text: string) => string; open: (sealed: string) => string }; + +function key(data: string, env: Record): Buffer { + if (env.ZOMB_SECRET_KEY) { + const k = /^[0-9a-f]{64}$/i.test(env.ZOMB_SECRET_KEY) ? Buffer.from(env.ZOMB_SECRET_KEY, 'hex') : Buffer.from(env.ZOMB_SECRET_KEY, 'base64'); + if (k.length !== 32) throw new Error('ZOMB_SECRET_KEY must be 32 bytes, as 64 hex characters or base64'); + return k; + } + const file = path.join(data, 'secret.key'); + if (!existsSync(file)) writeFileSync(file, randomBytes(32).toString('base64'), { mode: 0o600, flag: 'wx' }); + return Buffer.from(readFileSync(file, 'utf8').trim(), 'base64'); +} + +export function vault(data: string, env: Record = process.env): Vault { + const k = key(data, env); + return { + seal(text) { + const iv = randomBytes(12); + const c = createCipheriv('aes-256-gcm', k, iv); + const body = Buffer.concat([c.update(text, 'utf8'), c.final()]); + return ['v1', iv, c.getAuthTag(), body].map((p) => (typeof p === 'string' ? p : p.toString('base64url'))).join(':'); + }, + open(sealed) { + const [v, iv, tag, body] = sealed.split(':'); + if (v !== 'v1') throw new Error('unknown secret format'); + const d = createDecipheriv('aes-256-gcm', k, Buffer.from(iv, 'base64url')); + d.setAuthTag(Buffer.from(tag, 'base64url')); + return Buffer.concat([d.update(Buffer.from(body, 'base64url')), d.final()]).toString('utf8'); + }, + }; +} diff --git a/src/traffic.ts b/src/traffic.ts index 1e7be7d..cc83085 100644 --- a/src/traffic.ts +++ b/src/traffic.ts @@ -79,18 +79,22 @@ export function readLine(line: string): Hit | null { return null; } -/** Add what one file (or a webhook body) says to `into`. A file that is one JSON array is read whole. */ -export function addText(into: Traffic, text: string) { +/** The requests in one file or webhook body: a JSON array, or one request per line. */ +export function hitsOf(text: string): Hit[] { const trimmed = text.trimStart(); if (trimmed.startsWith('[')) { try { - for (const o of JSON.parse(trimmed)) add(into, fromObject(o)); - return; + const list = JSON.parse(trimmed); + if (Array.isArray(list)) return list.map(fromObject).filter((h): h is Hit => h !== null); } catch { - // not one array after all: fall through to lines + // not one array after all: read it as lines } } - for (const line of text.split('\n')) add(into, readLine(line)); + return text.split('\n').map(readLine).filter((h): h is Hit => h !== null); +} + +export function addText(into: Traffic, text: string) { + for (const hit of hitsOf(text)) add(into, hit); } function add(into: Traffic, hit: Hit | null) { diff --git a/test/cloud.test.ts b/test/cloud.test.ts index 450adc3..17dc8bb 100644 --- a/test/cloud.test.ts +++ b/test/cloud.test.ts @@ -7,6 +7,7 @@ import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from 'node:fs'; import { createServer } from 'node:http'; import { tmpdir } from 'node:os'; import path from 'node:path'; +import { gzipSync } from 'node:zlib'; import { start } from '../cloud/server.ts'; const stripeKey = ['sk', 'live', 'Q7mZp2Lx9Rt4Vb8Nc1Hs6Kd3'].join('_'); @@ -35,6 +36,20 @@ function fakeGitHub(): Promise<{ server: any; calls: any[]; url: string }> { if (req.method === 'GET' && call.path === '/repos/o/r') return reply(200, { clone_url: server.cloneUrl, default_branch: 'main' }); if (req.method === 'GET' && call.path === '/repos/o/site') return reply(200, { clone_url: server.siteUrl, default_branch: 'main' }); if (req.method === 'GET' && call.path === '/repos/o/gone') return reply(200, { clone_url: 'https://gone.invalid/o/gone.git', default_branch: 'main' }); + // a Next.js site whose weekly issue goes to Copilot + if (req.method === 'GET' && call.path === '/repos/o/web') return reply(200, { clone_url: server.webUrl, default_branch: 'main' }); + if (req.method === 'GET' && call.path.startsWith('/repos/o/web/issues?')) return reply(200, []); + if (req.method === 'POST' && call.path === '/repos/o/web/issues') return reply(201, { number: 8, assignees: [] }); + if (req.method === 'PATCH' && call.path === '/repos/o/web/issues/8') return reply(200, { number: 8, assignees: [] }); + if (req.method === 'POST' && call.path === '/repos/o/web/issues/8/assignees') return reply(201, { number: 8 }); + // sign-in: a code, then refresh tokens that work once each + if (req.method === 'POST' && call.path === '/login/oauth/access_token') { + if (call.body.code) return reply(200, { access_token: 'user-1', expires_in: 60, refresh_token: 'refresh-1', refresh_token_expires_in: 15897600 }); + if (call.body.grant_type === 'refresh_token' && call.body.refresh_token === 'refresh-1') return reply(200, { access_token: 'user-2', expires_in: 28800, refresh_token: 'refresh-2', refresh_token_expires_in: 15897600 }); + return reply(200, { error: 'bad_refresh_token', error_description: 'The refresh token passed is incorrect or expired.' }); + } + if (req.method === 'GET' && call.path === '/user') return reply(200, { login: 'dev' }); + if (req.method === 'GET' && call.path.startsWith('/user/installations')) return reply(200, { installations: [{ id: 1 }] }); reply(404, { message: `unexpected ${req.method} ${call.path}` }); }); return new Promise((r) => server.listen(0, () => r({ server, calls, url: `http://localhost:${server.address().port}` }))); @@ -127,6 +142,138 @@ test('pull requests get a check and a comment; the weekly clean-up opens an issu } }); +test('a log drain feeds the weekly scan, and the weekly issue goes to Copilot by itself', async () => { + const dir = mkdtempSync(path.join(tmpdir(), 'zomb-cloud-traffic-')); + const gh = await fakeGitHub(); + let app: any; + try { + const work = path.join(dir, 'web'); + const files = { + 'package.json': JSON.stringify({ name: 'web', private: true, dependencies: { next: '15.0.0', react: '19.0.0' } }), + 'app/page.tsx': 'export default function Home() { return About; }\n', + 'app/about/page.tsx': 'export default function About() { return

about

; }\n', + 'app/promo/page.tsx': 'export default function Promo() { return

promo

; }\n', + 'app/old/page.tsx': 'export default function Old() { return

old

; }\n', + 'app/blog/[slug]/page.tsx': 'export default function Post() { return

post

; }\n', + '.zomb/config.yml': 'handoff: copilot\n', + }; + for (const [f, body] of Object.entries(files)) { + mkdirSync(path.dirname(path.join(work, f)), { recursive: true }); + writeFileSync(path.join(work, f), body); + } + const git = (cwd, ...args) => execFileSync('git', ['-c', 'user.name=Dev', '-c', 'user.email=dev@example.com', ...args], { cwd, encoding: 'utf8' }).trim(); + git(work, 'init', '-q', '-b', 'main'); + git(work, 'add', '-A'); + git(work, 'commit', '-q', '-m', 'init'); + gh.server.webUrl = `file://${work}`; + + const secret = 'shh'; + app = await start({ data: path.join(dir, 'data'), url: 'http://localhost', port: 0, appId: '123', slug: 'zomb', privateKey, webhookSecret: secret, clientId: 'c', clientSecret: 's', api: gh.url, web: gh.url }, { log: () => {} }); + const base = `http://localhost:${app.port}`; + const hook = (event, payload) => { + const body = JSON.stringify(payload); + return fetch(`${base}/webhook`, { method: 'POST', body, headers: { 'x-github-event': event, 'x-hub-signature-256': `sha256=${createHmac('sha256', secret).update(body).digest('hex')}` } }); + }; + await hook('installation', { action: 'created', installation: { id: 1 }, repositories: [{ id: 50, full_name: 'o/web' }] }); + await app.idle(); + app.db.prepare('INSERT INTO sessions (id, login, token, installations, expires) VALUES (?, ?, ?, ?, ?)').run('sess', 'dev', 'session-token', '[1]', Date.now() + 3600_000); + const as = (p: string, init: RequestInit = {}) => fetch(`${base}${p}`, { redirect: 'manual', ...init, headers: { cookie: 'zomb=sess', origin: 'http://localhost', ...(init.headers || {}) } }); + + // the settings page hands out the drain's endpoint and token + const settings = await (await as('/repos/50')).text(); + assert.match(settings, /http:\/\/localhost\/drain\/50/); + const token = settings.match(/Bearer ([\w-]+)/)![1]; + assert.equal((await fetch(`${base}/repos/50`, { redirect: 'manual' })).status, 302, 'signed-out visitors are sent to sign in'); + + // Vercel signs a JSON array; anything else sends the token, gzipped or not + const day = (ago: number) => Date.now() - ago * 864e5; + const vercel = Buffer.from(JSON.stringify([{ source: 'lambda', path: '/blog/hello', timestamp: day(20) }, { proxy: { path: '/_next/static/chunk.js' }, timestamp: day(20) }])); + const drain = (body: Buffer, headers: Record) => fetch(`${base}/drain/50`, { method: 'POST', body, headers }); + assert.equal((await drain(vercel, { 'x-vercel-signature': 'nope' })).status, 401, 'unsigned drains are refused'); + const signed = await drain(vercel, { 'x-vercel-signature': createHmac('sha1', token).update(vercel).digest('hex') }); + assert.deepEqual(await signed.json(), { requests: 1, dropped: 0 }, 'the static file is not a request for a route'); + const lines = gzipSync(`${JSON.stringify({ path: '/about?ref=home', timestamp: day(3) })}\n${JSON.stringify({ path: '/promo', timestamp: day(1) })}\n`); + assert.equal((await drain(lines, { authorization: `Bearer ${token}`, 'content-encoding': 'gzip' })).status, 200); + const rows = app.db.prepare('SELECT path, hits FROM traffic WHERE repo = 50 ORDER BY path').all().map((r) => `${r.path} ${r.hits}`); + assert.deepEqual(rows, ['/about 1', '/blog/hello 1', '/promo 1'], 'paths and counts only, static files skipped'); + assert.equal((await drain(Buffer.from('/x'), { authorization: 'Bearer x' })).status, 401); + assert.equal((await fetch(`${base}/drain/999`, { method: 'POST', body: '/x', headers: { authorization: `Bearer ${token}` } })).status, 404); + + // turning on automatic hand-off signs in again, and keeps that token sealed + const on = await as('/repos/50/copilot/auto', { method: 'POST', body: 'on=1', headers: { 'content-type': 'application/x-www-form-urlencoded' } }); + assert.equal(on.status, 302); + assert.match(on.headers.get('location')!, /\/login\/oauth\/authorize\?client_id=c/); + const jar = on.headers.getSetCookie().map((c) => c.split(';')[0]); + const state = jar.find((c) => c.startsWith('zomb_state='))!.split('=')[1]; + const back = await fetch(`${base}/callback?code=abc&state=${state}`, { redirect: 'manual', headers: { cookie: jar.join('; ') } }); + assert.equal(back.headers.get('location'), '/repos/50'); + assert.equal(app.db.prepare('SELECT copilot_login FROM repos WHERE id = 50').get().copilot_login, 'dev'); + const grant = JSON.stringify(app.db.prepare('SELECT * FROM grants').all()); + assert.ok(grant.includes('"dev"') && !grant.includes('user-1') && !grant.includes('refresh-1'), `tokens are sealed at rest: ${grant}`); + assert.match(await (await as('/')).text(), /Copilot, automatic/); + + // the weekly issue: routes no request reached, then assigned to Copilot with a refreshed token + assert.equal(await app.weekly(50), 8); + const issue = gh.calls.find((c) => c.method === 'POST' && c.path === '/repos/o/web/issues'); + assert.match(issue.body.body, /Delete \/old: no links and no traffic/); + assert.doesNotMatch(issue.body.body, /\/promo|\/about|\/blog/, 'routes with requests are in use, linked or not'); + assert.match(issue.body.body, /assigns this issue to Copilot every week, on behalf of dev/); + const refresh = gh.calls.find((c) => c.path === '/login/oauth/access_token' && c.body.grant_type === 'refresh_token'); + assert.equal(refresh?.body.refresh_token, 'refresh-1', 'the short-lived token is refreshed before use'); + const assign = gh.calls.find((c) => c.method === 'POST' && c.path === '/repos/o/web/issues/8/assignees'); + assert.equal(assign.auth, 'Bearer user-2', 'assigned as the person, with the fresh token'); + assert.deepEqual(assign.body.assignees, ['copilot-swe-agent[bot]']); + + // revoking zomb on GitHub stops the hand-off and says so + await hook('github_app_authorization', { action: 'revoked', sender: { login: 'dev' } }); + assert.equal(app.db.prepare('SELECT count(*) AS n FROM grants').get().n, 0); + assert.match(await (await as('/')).text(), /Copilot stopped/); + assert.match(await (await as('/repos/50')).text(), /Stopped: dev revoked/); + } finally { + await app?.close(); + gh.server.close(); + rmSync(dir, { recursive: true, force: true }); + } +}); + +test('a hand-off GitHub refuses puts the manual line back and asks for a new sign-in', async () => { + const { jobs } = await import('../cloud/jobs.ts'); + const { DatabaseSync } = await import('node:sqlite'); + const db = new DatabaseSync(':memory:'); + db.exec("CREATE TABLE repos (id INTEGER PRIMARY KEY, installation INTEGER, name TEXT, weekly_at TEXT, note TEXT, drain_secret TEXT, copilot_login TEXT, handoff_error TEXT); CREATE TABLE scans (id INTEGER PRIMARY KEY AUTOINCREMENT, repo INTEGER, sha TEXT, kind TEXT, at TEXT, summary TEXT, failing INTEGER, tasks INTEGER); CREATE TABLE traffic (repo INTEGER, day TEXT, path TEXT, hits INTEGER)"); + const dir = mkdtempSync(path.join(tmpdir(), 'zomb-cloud-refused-')); + try { + mkdirSync(path.join(dir, 'src')); + writeFileSync(path.join(dir, 'package.json'), '{"name":"r","private":true}'); + writeFileSync(path.join(dir, 'src/index.ts'), 'export const a = 1;\n'); + writeFileSync(path.join(dir, 'src/old.ts'), 'export const b = 2;\n'); + mkdirSync(path.join(dir, '.zomb')); + writeFileSync(path.join(dir, '.zomb/config.yml'), 'handoff: copilot\n'); + execFileSync('git', ['init', '-q', '-b', 'main'], { cwd: dir }); + execFileSync('git', ['add', '-A'], { cwd: dir }); + execFileSync('git', ['-c', 'user.name=D', '-c', 'user.email=d@e.com', 'commit', '-q', '-m', 'i'], { cwd: dir }); + db.prepare("INSERT INTO repos (id, installation, name, copilot_login) VALUES (1, 1, 'o/r', 'dev')").run(); + const calls: any[] = []; + const api = async (method: string, url: string, body?: any) => { + calls.push({ method, url, body }); + if (url === '/repos/o/r') return { clone_url: `file://${dir}`, default_branch: 'main' }; + if (url.startsWith('/repos/o/r/issues?')) return []; + return { number: 3, assignees: [] }; + }; + let dropped = ''; + const grants: any = { has: () => true, drop: (l: string) => (dropped = l), token: async () => Promise.reject(Object.assign(new Error('bad'), { code: 'bad_refresh_token' })) }; + const work = jobs({ gh: { as: () => api, user: () => api, installationToken: async () => '' }, db, grants, log: () => {} }); + assert.equal(await work.weekly(db.prepare('SELECT * FROM repos').get()), 3); + const bodies = calls.filter((c) => c.body?.body).map((c) => c.body.body); + assert.match(bodies[0], /on behalf of dev/); + assert.match(bodies.at(-1), /Assign this issue to Copilot from the zomb dashboard/, 'the issue no longer claims it was handed over'); + assert.equal(dropped, 'dev'); + assert.match((db.prepare('SELECT handoff_error FROM repos').get() as any).handoff_error, /no longer accepts dev/); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + test('the field places your repos among the study authors, biggest first', async () => { const { field } = await import('../cloud/server.ts'); const authors = { Human: { deletedPer100: 33.07 }, Claude: { deletedPer100: 20.77 }, Devin: { deletedPer100: 36.6 } }; From 31270c2de9182e4c10702487160ccaac09924629 Mon Sep 17 00:00:00 2001 From: Diip3sh Date: Tue, 13 Oct 2026 12:58:04 +0530 Subject: [PATCH 4/7] Cloud: scan untrusted repos in a throwaway container ZOMB_SANDBOX=docker runs each job's install and scan in a container of the zomb-cloud image, as uid 1000 with no capabilities, no new privileges, 2 GB, 2 CPUs and 512 processes, for at most 10 minutes. The server still fetches the repo, so the GitHub token never enters the container; the checkout (and its traffic file) arrives as a tar on stdin, which also works when the server runs in Docker on the host's socket. Nothing is mounted. ZOMB_SANDBOX_IMAGE and ZOMB_SANDBOX_NETWORK pick the image and the network; the server checks the image at startup. The image now carries the Docker CLI and a /work folder for uid 1000, makes its sources readable whatever their modes in the build context, and starts cloud/server.ts. A .dockerignore keeps node_modules and .git out of the build context. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- .dockerignore | 5 +++ cloud/Dockerfile | 8 ++++- cloud/README.md | 20 ++++++++++-- cloud/jobs.ts | 79 ++++++++++++++++++++++++++++++++++++++++------ cloud/server.ts | 23 +++++++++----- test/cloud.test.ts | 68 +++++++++++++++++++++++++++++++++++++++ 6 files changed, 182 insertions(+), 21 deletions(-) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..1779a30 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,5 @@ +.git +node_modules +dist +cloud/data +*.profraw diff --git a/cloud/Dockerfile b/cloud/Dockerfile index 273a336..8807701 100644 --- a/cloud/Dockerfile +++ b/cloud/Dockerfile @@ -1,12 +1,18 @@ # zomb Cloud: docker build -f cloud/Dockerfile -t zomb-cloud . && docker run -p 3000:3000 -v zomb:/data -e ZOMB_URL=https://your.host zomb-cloud FROM node:26-slim +# the Docker CLI, for ZOMB_SANDBOX=docker: the server starts each scan in a throwaway container of this same image +COPY --from=docker:cli /usr/local/bin/docker /usr/local/bin/docker RUN apt-get update && apt-get install -y --no-install-recommends git ca-certificates && rm -rf /var/lib/apt/lists/* && npm install -g pnpm yarn +# where a sandboxed scan unpacks its checkout, as the image's unprivileged node user (uid 1000) +RUN mkdir /work && chown node:node /work WORKDIR /app COPY package.json package-lock.json ./ RUN npm ci --omit=dev COPY src src COPY cloud cloud +# a sandboxed scan runs /app/src/cli.ts as uid 1000, whatever modes the files had in the build context +RUN chmod -R a+rX src cloud ENV DATA_DIR=/data PORT=3000 VOLUME /data EXPOSE 3000 -CMD ["node", "cloud/server.js"] +CMD ["node", "cloud/server.ts"] diff --git a/cloud/README.md b/cloud/README.md index 478d346..8b10894 100644 --- a/cloud/README.md +++ b/cloud/README.md @@ -10,7 +10,7 @@ A GitHub App around the zomb CLI. Install it and you're done: no workflow file. ## Run it ```bash -ZOMB_URL=https://zomb.example.com node cloud/server.js +ZOMB_URL=https://zomb.example.com node cloud/server.ts ``` or with Docker: @@ -53,8 +53,22 @@ Copilot only takes an assignment from a person, never from an app. On a repo's s The token is encrypted at rest with AES-256-GCM. The key comes from `ZOMB_SECRET_KEY` (32 bytes, as hex or base64), or from a key generated into `DATA_DIR/secret.key`. The generated key sits next to the database, so it protects copies of the database (backups, dumps), not the disk. Set `ZOMB_SECRET_KEY` to keep the two apart. GitHub user tokens last 8 hours, so zomb refreshes them with the refresh token, which GitHub replaces on every use. If GitHub refuses the token, or you revoke zomb in your GitHub settings, zomb deletes it, puts the manual line back on the issue, and the dashboard shows **Copilot stopped** until someone turns it on again. Turning it off on every repo deletes the token. -## Before you host it for other people +## Scans in containers -Scanning installs a repo's dependencies (with `--ignore-scripts`) and Knip loads its config files, so a scan runs some of that repo's code. Jobs get a clean environment with no server secrets, and the GitHub token never touches the checkout, but they still run as the server's user. That's fine for your own repos. Before scanning code you don't trust, run each job in its own container. The place to do that is `checkout()` in `cloud/jobs.js`. +Scanning installs a repo's dependencies (with `--ignore-scripts`) and Knip loads its config files, so a scan runs some of that repo's code. By default that happens as the server's user, in a clean environment with no server secrets. That's fine for your own repos. + +Before you scan code you don't trust, set `ZOMB_SANDBOX=docker`. The server still fetches each repo itself, so the GitHub token never enters a container. It streams the checkout into a throwaway container, where the install and the scan run as an unprivileged user with no capabilities and no way to gain privileges, limited to 2 GB of memory, 2 CPUs and 512 processes, for at most 10 minutes. The container is removed when the scan ends. + +- `ZOMB_SANDBOX_IMAGE`: the image scans run in, `zomb-cloud` by default. The Dockerfile above builds it, Docker CLI included. The server checks at startup that the image exists. +- `ZOMB_SANDBOX_NETWORK`: the Docker network scan containers join. Installing needs the package registry, so by default they use Docker's bridge network, which reaches whatever the host reaches. Give them a network that only reaches your registry (or a mirror) to keep scans out of your private network. With `none`, installs fail and Knip reads each repo without its dependencies, so some findings get less precise. + +To run the server itself in Docker with the sandbox on, give it the host's Docker socket: + +```bash +docker run -p 3000:3000 -v zomb:/data -v /var/run/docker.sock:/var/run/docker.sock \ + -e ZOMB_SANDBOX=docker -e ZOMB_URL=https://zomb.example.com zomb-cloud +``` + +Whoever holds that socket controls the host. Only the server container gets it; scan containers never do. If that trade is wrong for you, run the server straight on a host with Docker, or point it at a rootless Docker daemon (`DOCKER_HOST`). Not here yet: billing. diff --git a/cloud/jobs.ts b/cloud/jobs.ts index f60dfbe..4f893c5 100644 --- a/cloud/jobs.ts +++ b/cloud/jobs.ts @@ -1,5 +1,6 @@ // The work behind each GitHub event: check the code out, run zomb on it, and report back on GitHub. -import { execFile } from 'node:child_process'; +import { execFile, spawn } from 'node:child_process'; +import { randomBytes } from 'node:crypto'; import { existsSync, readFileSync, writeFileSync } from 'node:fs'; import { mkdir, mkdtemp, realpath, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; @@ -29,8 +30,8 @@ const WEEKLY = ''; const INSTALL = { npm: ['ci'], pnpm: ['install', '--frozen-lockfile'], yarn: ['install', '--frozen-lockfile'], bun: ['install', '--frozen-lockfile'] }; // Check out a repo into a fresh folder. The token is only ever in the fetch URL, never in .git/config, -// and nothing the repo runs sees the server's environment. -// ponytail: jobs are child processes of the server; give each its own container before scanning code you don't trust +// and nothing the repo runs sees the server's environment. Fetching stays here, so with the sandbox on +// the token never enters the container: only the checkout does. async function checkout(cloneUrl: string, token: string, refspecs: string[], ref: string) { const root = await realpath(await mkdtemp(path.join(tmpdir(), 'zomb-cloud-'))); const dir = path.join(root, 'repo'); @@ -45,10 +46,10 @@ async function checkout(cloneUrl: string, token: string, refspecs: string[], ref await run('git', ['checkout', '-q', ref]); // Knip needs the dependencies; --ignore-scripts so installing never runs the repo's own code const pm = packageManager(dir); - if (['package-lock.json', 'pnpm-lock.yaml', 'yarn.lock', 'bun.lock', 'bun.lockb'].some((f) => existsSync(path.join(dir, f)))) await run(pm, [...INSTALL[pm], '--ignore-scripts']).catch(() => {}); + const install = ['package-lock.json', 'pnpm-lock.yaml', 'yarn.lock', 'bun.lock', 'bun.lockb'].some((f) => existsSync(path.join(dir, f))) ? [pm, ...INSTALL[pm], '--ignore-scripts'] : null; const file = path.join(dir, '.zomb', 'config.yml'); const config = { 'fail-on': 'high', weekly: true, handoff: 'none', ...(existsSync(file) ? parse(readFileSync(file, 'utf8')) : {}) }; - return { root, dir, config, run, cleanup: () => rm(root, { recursive: true, force: true }) }; + return { root, dir, config, run, install, cleanup: () => rm(root, { recursive: true, force: true }) }; } catch (e) { await rm(root, { recursive: true, force: true }); // execFile puts the command in its message, and the fetch URL carries the token: it would reach the log and the check run @@ -56,16 +57,76 @@ async function checkout(cloneUrl: string, token: string, refspecs: string[], ref throw e; } } +type Checkout = Awaited>; + +/** ZOMB_SANDBOX=docker: install and scan each checkout in a throwaway container instead of as the server's user. */ +export type Sandbox = { image: string; network?: string } | null; +export const sandboxFrom = (env: Record): Sandbox => (env.ZOMB_SANDBOX === 'docker' ? { image: env.ZOMB_SANDBOX_IMAGE || 'zomb-cloud', network: env.ZOMB_SANDBOX_NETWORK } : null); +const WORK = '/work'; + +// `docker run` for one job. The checkout arrives as a tar on stdin, so this works the same when the server +// itself runs in a container and talks to the host's Docker (a bind mount would name a path on the host). +// Arguments reach the script as "$@", never through the shell; only the install command, built from fixed words, is inlined. +export function sandboxArgs({ name, image, network, install, args }: { name: string; image: string; network?: string; install: string[] | null; args: string[] }) { + // macOS tar adds extended-attribute headers the image's GNU tar would warn about on every scan + const script = ['set -e', `tar -x --warning=no-unknown-keyword -C ${WORK}`, `cd ${WORK}/repo`, ...(install ? [`${install.join(' ')} >/dev/null 2>&1 || true`] : []), `exec node /app/src/cli.ts ${WORK}/repo --json "$@"`].join('; '); + return [ + 'run', '--rm', '-i', '--name', name, + '--user', '1000:1000', '--cap-drop', 'ALL', '--security-opt', 'no-new-privileges', + '--memory', '2g', '--cpus', '2', '--pids-limit', '512', + ...(network ? ['--network', network] : []), + '-e', `HOME=${WORK}/home`, '-e', 'CI=1', '-e', 'NO_COLOR=1', + '--entrypoint', 'sh', image, '-c', script, 'zomb', ...args, + ]; +} + +function inContainer(co: Checkout, args: string[], sandbox: NonNullable, timeout = 600_000): Promise<{ code: number | null; stdout: string; stderr: string }> { + const name = `zomb-job-${randomBytes(6).toString('hex')}`; + // files beside the checkout (the traffic file) have the same place under /work + const inside = args.map((a) => (a.startsWith(`${co.root}/`) ? `${WORK}${a.slice(co.root.length)}` : a)); + return new Promise((resolve, reject) => { + const docker = spawn('docker', sandboxArgs({ name, image: sandbox.image, network: sandbox.network, install: co.install, args: inside }), { stdio: ['pipe', 'pipe', 'pipe'] }); + // COPYFILE_DISABLE: macOS tar would add a ._ file per file for its extended attributes, and the scan would count them + const tar = spawn('tar', ['-c', '-C', co.root, '.'], { stdio: ['ignore', 'pipe', 'ignore'], env: { PATH: process.env.PATH, COPYFILE_DISABLE: '1' } }); + let failed: Error | null = null; + const fail = (e: Error) => ((failed ??= e), docker.kill()); + tar.on('error', fail); + tar.on('close', (code) => code && fail(new Error(`could not pack the checkout (tar exited with ${code})`))); + tar.stdout.pipe(docker.stdin); + // the container may exit before it has read everything + docker.stdin.on('error', () => {}); + let stdout = ''; + let stderr = ''; + docker.stdout.on('data', (c) => (stdout += c)); + docker.stderr.on('data', (c) => (stderr += c)); + // killing the docker CLI leaves the container running: remove it by name + const timer = setTimeout(() => { + failed ??= new Error(`the scan took longer than ${timeout / 60_000} minutes`); + execFile('docker', ['rm', '-f', name], () => {}); + }, timeout); + docker.on('error', (e) => (clearTimeout(timer), tar.kill(), reject(failed ?? e))); + docker.on('close', (code) => (clearTimeout(timer), failed ? reject(failed) : resolve({ code, stdout, stderr }))); + }); +} // zomb --json on the checkout; --fail-on exits 1 but still prints the result -async function scan(co: { dir: string; run: (cmd: string, args: string[], timeout?: number) => Promise<{ stdout: string }> }, args: string[]): Promise { +async function runScan(co: Checkout, args: string[], sandbox: Sandbox): Promise { let why = ''; - const out = await co.run(process.execPath, [CLI, co.dir, '--json', ...args]).then((r) => r.stdout, (e: { stdout?: string; stderr?: string }) => ((why = e.stderr?.trim().split('\n').at(-1) || ''), e.stdout)); - if (!out) throw new Error(why.replace(co.dir, 'the repo') || 'zomb produced no result'); + let out: string | undefined; + if (sandbox) { + const r = await inContainer(co, args, sandbox); + out = r.stdout; + why = r.stderr.trim().split('\n').at(-1) || `docker exited with ${r.code}`; + } else { + if (co.install) await co.run(co.install[0], co.install.slice(1)).catch(() => {}); + out = await co.run(process.execPath, [CLI, co.dir, '--json', ...args]).then((r) => r.stdout, (e: { stdout?: string; stderr?: string }) => ((why = e.stderr?.trim().split('\n').at(-1) || ''), e.stdout)); + } + if (!out) throw new Error(why.replace(co.dir, 'the repo').replace(`${WORK}/repo`, 'the repo') || 'zomb produced no result'); return JSON.parse(out); } -export function jobs({ gh, db, appId, log = console.error, grants }: { gh: any; db: any; appId?: string | number; log?: (s: string) => void; grants?: Grants }) { +export function jobs({ gh, db, appId, log = console.error, grants, sandbox = null }: { gh: any; db: any; appId?: string | number; log?: (s: string) => void; grants?: Grants; sandbox?: Sandbox }) { + const scan = (co: Checkout, args: string[]) => runScan(co, args, sandbox); const record = (repo: number | string, sha: string | null, kind: string, result: Result) => db.prepare('INSERT INTO scans (repo, sha, kind, at, summary, failing, tasks) VALUES (?, ?, ?, ?, ?, ?, ?)').run(repo, sha, kind, new Date().toISOString(), JSON.stringify(result.summary), result.failing, result.tasks.length); diff --git a/cloud/server.ts b/cloud/server.ts index 565495d..831dff7 100644 --- a/cloud/server.ts +++ b/cloud/server.ts @@ -2,6 +2,7 @@ // zomb Cloud: a GitHub App. Install it and every pull request gets a zomb check and comment (no workflow file), // every push to the default branch feeds the trends dashboard, and each repo gets a weekly clean-up issue // handed to the agent the team already uses. Self-host it, or use ours. +import { execFile } from 'node:child_process'; import { createHmac, randomBytes, timingSafeEqual } from 'node:crypto'; import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; import { createServer } from 'node:http'; @@ -12,7 +13,8 @@ import { gunzipSync } from 'node:zlib'; import { hitsOf } from '../src/traffic.ts'; import { github, verify } from './github.ts'; import { grants } from './grants.ts'; -import { assignCopilot, jobs, TRAFFIC_MIN_DAYS } from './jobs.ts'; +import { assignCopilot, jobs, sandboxFrom, TRAFFIC_MIN_DAYS } from './jobs.ts'; +import type { Sandbox } from './jobs.ts'; import { vault } from './vault.ts'; type Session = { id: string; login: string; token: string; installations: number[]; expires: number }; @@ -40,10 +42,12 @@ function config(env: Record = process.env) { clientSecret: env.GITHUB_CLIENT_SECRET || saved.client_secret, api: env.GITHUB_API || 'https://api.github.com', web: env.GITHUB_WEB || 'https://github.com', + sandbox: sandboxFrom(env), }; } +type Config = Omit, 'sandbox'> & { sandbox?: Sandbox }; -export function start(cfg: ReturnType = config(), { log = console.error }: { log?: (s: string) => void } = {}) { +export function start(cfg: Config = config(), { log = console.error }: { log?: (s: string) => void } = {}) { mkdirSync(cfg.data, { recursive: true }); const db = new DatabaseSync(path.join(cfg.data, 'zomb.db')); db.exec(`CREATE TABLE IF NOT EXISTS repos (id INTEGER PRIMARY KEY, installation INTEGER NOT NULL, name TEXT NOT NULL, weekly_at TEXT); @@ -60,7 +64,7 @@ export function start(cfg: ReturnType = config(), { log = console const gh = github(cfg); const secrets = vault(cfg.data); const tokens = grants({ db, vault: secrets, gh }); - const work = jobs({ gh, db, appId: cfg.appId, log, grants: tokens }); + const work = jobs({ gh, db, appId: cfg.appId, log, grants: tokens, sandbox: cfg.sandbox }); const forget = (repo: number) => db.prepare('DELETE FROM traffic WHERE repo = ?').run(repo); // ponytail: in-memory queue, two jobs at a time; a restart drops queued jobs (the next push or week redoes them) @@ -388,7 +392,7 @@ const page = (title: string, body: string) => `
${body}
`; -function landing(cfg: ReturnType) { +function landing(cfg: Config) { return page('zomb', `

zomb cloud

A health check on every pull request.

Trends across your repos, and a weekly clean-up handed to the agent your team already pays for.

Install on GitHubSign in
`); @@ -513,7 +517,7 @@ export function field(yours: number | null, authors: Record`; } -function dashboard(db: any, me: Session, cfg: ReturnType, view = 'all') { +function dashboard(db: any, me: Session, cfg: Config, view = 'all') { const repos = (db.prepare(`SELECT * FROM repos WHERE installation IN (${me.installations.map(() => '?').join(',') || 'NULL'}) ORDER BY name`).all(...me.installations) as any[]).map((r) => { const scans = (db.prepare("SELECT * FROM scans WHERE repo = ? AND kind != 'pr' ORDER BY id DESC LIMIT 12").all(r.id) as any[]).reverse().map((s) => ({ ...s, summary: JSON.parse(s.summary) })); return { ...r, scans, last: scans.at(-1)?.summary, prev: scans.at(-2)?.summary, at: scans.at(-1)?.at }; @@ -613,7 +617,7 @@ function trafficStats(db: any, repo: number) { } // One repo's settings: where its log drain posts, and whether its weekly issue goes to Copilot by itself. -function settingsPage(cfg: ReturnType, repo: any, secret: string, stats: ReturnType, granted: boolean) { +function settingsPage(cfg: Config, repo: any, secret: string, stats: ReturnType, granted: boolean) { const endpoint = `${cfg.url}/drain/${repo.id}`; const enough = stats.first !== null && Date.parse(stats.first) <= Date.now() - TRAFFIC_MIN_DAYS * 864e5; const status = stats.hits @@ -646,7 +650,7 @@ ${copilot} } // Create the GitHub App from a manifest: one click on GitHub, and the credentials come back here. -function setupPage(cfg: ReturnType, ready: boolean) { +function setupPage(cfg: Config, ready: boolean) { if (ready) return page('Set up', '

Already set up

Go to the dashboard

'); const manifest = { name: 'zomb', @@ -665,5 +669,8 @@ function setupPage(cfg: ReturnType, ready: boolean) { if (import.meta.url === `file://${process.argv[1]}`) { const cfg = config(); const s = await start(cfg); - console.error(`zomb cloud on ${cfg.url} (port ${s.port})${cfg.appId ? '' : `: open ${cfg.url}/setup to create the GitHub App`}`); + console.error(`zomb cloud on ${cfg.url} (port ${s.port})${cfg.sandbox ? `, scans in Docker (${cfg.sandbox.image})` : ''}${cfg.appId ? '' : `: open ${cfg.url}/setup to create the GitHub App`}`); + // every scan would fail on a missing image: say so now, not on the first push + const image = cfg.sandbox?.image; + if (image) execFile('docker', ['image', 'inspect', image], (e, _, stderr) => e && console.error(`ZOMB_SANDBOX=docker, but scans cannot start: ${stderr.trim() || e.message}`)); } diff --git a/test/cloud.test.ts b/test/cloud.test.ts index 17dc8bb..31eff7b 100644 --- a/test/cloud.test.ts +++ b/test/cloud.test.ts @@ -274,6 +274,74 @@ test('a hand-off GitHub refuses puts the manual line back and asks for a new sig } }); +test('a sandboxed scan runs unprivileged, mounts nothing, and keeps its arguments out of the script', async () => { + const { sandboxArgs, sandboxFrom } = await import('../cloud/jobs.ts'); + assert.equal(sandboxFrom({}), null, 'off unless asked for'); + assert.deepEqual(sandboxFrom({ ZOMB_SANDBOX: 'docker', ZOMB_SANDBOX_NETWORK: 'scans' }), { image: 'zomb-cloud', network: 'scans' }); + const args = sandboxArgs({ name: 'job', image: 'img', install: ['npm', 'ci', '--ignore-scripts'], args: ['--since', 'origin/x; rm -rf /'] }); + const flag = (f: string) => args[args.indexOf(f) + 1]; + assert.equal(flag('--user'), '1000:1000'); + assert.equal(flag('--cap-drop'), 'ALL'); + assert.equal(flag('--security-opt'), 'no-new-privileges'); + assert.ok(args.includes('--rm') && !args.includes('--network'), 'thrown away after, on the default network unless one is set'); + assert.ok(!args.some((a) => /^(-v|--volume|--mount|--privileged)$|docker\.sock/.test(a)), `no mounts: ${args.join(' ')}`); + const script = flag('-c'); + assert.match(script, /npm ci --ignore-scripts/); + assert.doesNotMatch(script, /origin\/x/); + assert.deepEqual(args.slice(-3), ['zomb', '--since', 'origin/x; rm -rf /'], 'arguments arrive as "$@"'); + const offline = sandboxArgs({ name: 'j', image: 'i', network: 'none', install: null, args: [] }); + assert.equal(offline[offline.indexOf('--network') + 1], 'none'); + assert.doesNotMatch(offline[offline.indexOf('-c') + 1], /--ignore-scripts/, 'no lockfile, no install'); +}); + +const sandboxImage = process.env.ZOMB_SANDBOX_IMAGE || 'zomb-cloud'; +const noSandbox = (() => { + try { + execFileSync('docker', ['image', 'inspect', sandboxImage], { stdio: 'ignore' }); + return false; + } catch { + return `needs Docker and the ${sandboxImage} image: docker build -f cloud/Dockerfile -t ${sandboxImage} .`; + } +})(); + +test('in the sandbox, a scan finds what it finds on the host, traffic included', { skip: noSandbox }, async () => { + const { jobs } = await import('../cloud/jobs.ts'); + const { DatabaseSync } = await import('node:sqlite'); + const db = new DatabaseSync(':memory:'); + db.exec('CREATE TABLE repos (id INTEGER PRIMARY KEY, installation INTEGER, name TEXT, note TEXT); CREATE TABLE scans (id INTEGER PRIMARY KEY AUTOINCREMENT, repo INTEGER, sha TEXT, kind TEXT, at TEXT, summary TEXT, failing INTEGER, tasks INTEGER); CREATE TABLE traffic (repo INTEGER, day TEXT, path TEXT, hits INTEGER)'); + const dir = mkdtempSync(path.join(tmpdir(), 'zomb-cloud-sandbox-')); + try { + const files = { + 'package.json': JSON.stringify({ name: 'web', private: true, dependencies: { next: '15.0.0', react: '19.0.0' } }), + 'app/page.tsx': 'export default function Home() { return About; }\n', + 'app/about/page.tsx': 'export default function About() { return

about

; }\n', + 'app/old/page.tsx': 'export default function Old() { return

old

; }\n', + 'lib/unused.ts': 'export const nobody = 1;\n', + }; + for (const [f, body] of Object.entries(files)) { + mkdirSync(path.dirname(path.join(dir, f)), { recursive: true }); + writeFileSync(path.join(dir, f), body); + } + execFileSync('git', ['init', '-q', '-b', 'main'], { cwd: dir }); + execFileSync('git', ['add', '-A'], { cwd: dir }); + execFileSync('git', ['-c', 'user.name=D', '-c', 'user.email=d@e.com', 'commit', '-q', '-m', 'i'], { cwd: dir }); + db.prepare("INSERT INTO repos (id, installation, name) VALUES (1, 1, 'o/web')").run(); + const day = (ago: number) => new Date(Date.now() - ago * 864e5).toISOString().slice(0, 10); + db.prepare("INSERT INTO traffic VALUES (1, ?, '/', 4), (1, ?, '/about', 2)").run(day(20), day(2)); + const api = async () => ({ clone_url: `file://${dir}`, default_branch: 'main' }); + const gh = { as: () => api, installationToken: async () => '' }; + const repo = db.prepare('SELECT * FROM repos').get(); + await jobs({ gh, db, log: () => {} }).first(repo); + await jobs({ gh, db, log: () => {}, sandbox: { image: sandboxImage } }).first(repo); + const [host, boxed] = db.prepare('SELECT summary, tasks FROM scans ORDER BY id').all().map((r: any) => ({ ...r, summary: JSON.parse(r.summary) })); + assert.equal(boxed.summary.traffic?.requests, 6, 'the traffic file reached the container'); + assert.ok(boxed.summary.zombie.files >= 1, 'the unused file is found'); + assert.deepEqual(boxed, host); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + test('the field places your repos among the study authors, biggest first', async () => { const { field } = await import('../cloud/server.ts'); const authors = { Human: { deletedPer100: 33.07 }, Claude: { deletedPer100: 20.77 }, Devin: { deletedPer100: 36.6 } }; From ea92d003b53e322091e2011f1f7d485a85f6593c Mon Sep 17 00:00:00 2001 From: Diip3sh Date: Tue, 13 Oct 2026 12:59:38 +0530 Subject: [PATCH 5/7] Point the last .js names at the .ts files Knip's entries in package.json named cloud/server.js, research/*.js and test/*.test.js, which no longer exist; the research usage lines and a comment in blueprint.ts did too. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- package.json | 7 +++---- research/page.ts | 2 +- research/state-of-ai-code.ts | 8 ++++---- src/blueprint.ts | 2 +- 4 files changed, 9 insertions(+), 10 deletions(-) diff --git a/package.json b/package.json index e7c9421..d5b614f 100644 --- a/package.json +++ b/package.json @@ -37,10 +37,9 @@ ], "entry": [ "src/bin.ts", - "cloud/server.js", - "research/*.js", - "test/*.test.js", - "research/page.ts" + "cloud/server.ts", + "research/*.ts", + "test/*.test.ts" ] }, "devDependencies": { diff --git a/research/page.ts b/research/page.ts index 51df526..c2826e7 100644 --- a/research/page.ts +++ b/research/page.ts @@ -1,5 +1,5 @@ // The State of AI Code as one static page, rendered from research/data/summary.json. -// node research/page.js > research/data/state-of-ai-code.html +// node research/page.ts > research/data/state-of-ai-code.html import { readFileSync } from 'node:fs'; import path from 'node:path'; diff --git a/research/state-of-ai-code.ts b/research/state-of-ai-code.ts index bf16050..9bdb90c 100644 --- a/research/state-of-ai-code.ts +++ b/research/state-of-ai-code.ts @@ -3,9 +3,9 @@ // Finds public JS/TS repos with agent-written commits, measures every commit in them (agents and the humans // working in the same repos), and compares agents per 1,000 lines they added. // -// node research/state-of-ai-code.js find [--per-agent 60] GitHub commit search -> research/data/repos.json -// node research/state-of-ai-code.js measure [--jobs 4] clone, measure, delete (resumable) -> research/data/repos/ -// node research/state-of-ai-code.js summary per-agent rates -> research/data/summary.json +// node research/state-of-ai-code.ts find [--per-agent 60] GitHub commit search -> research/data/repos.json +// node research/state-of-ai-code.ts measure [--jobs 4] clone, measure, delete (resumable) -> research/data/repos/ +// node research/state-of-ai-code.ts summary per-agent rates -> research/data/summary.json // // Secrets are only counted, never stored. Needs the gh CLI, logged in. import { execFile, spawn } from 'node:child_process'; @@ -304,5 +304,5 @@ if (import.meta.url === `file://${process.argv[1]}`) { if (step === 'find') await find(Number(values['per-agent'])); else if (step === 'measure') await measureAll(Number(values.jobs)); else if (step === 'summary') await summary(); - else console.log('usage: node research/state-of-ai-code.js find | measure | summary'); + else console.log('usage: node research/state-of-ai-code.ts find | measure | summary'); } diff --git a/src/blueprint.ts b/src/blueprint.ts index 05a4f05..12f1e53 100644 --- a/src/blueprint.ts +++ b/src/blueprint.ts @@ -8,7 +8,7 @@ import { truthy } from './types.ts'; import type { BrokenRule, Parsed, Rule, RouteFacts, Rules } from './types.ts'; export const FILE = '.zomb/blueprint.yml'; -// short names for the jobs in score.js, as they read in the YAML +// short names for the jobs in score.ts, as they read in the YAML const JOBS: Record = { 'icon sets': 'icons', 'date libraries': 'dates', 'HTTP clients': 'http', 'animation libraries': 'animation', 'state managers': 'state', 'form libraries': 'forms', 'validation libraries': 'validation', 'CSS-in-JS libraries': 'css', 'toast libraries': 'toasts', 'chart libraries': 'charts', 'UI kits': 'ui', 'markdown renderers': 'markdown', 'data-fetching libraries': 'fetching', ORMs: 'orm' }; const HOOK = /(^|\/)use[A-Z]\w*\.[cm]?[jt]sx?$/; const SHARED = /^(utils?|libs?|helpers?|common|shared|core)$/i; From b54dff89cc17643b23d038229b3b1a2425b146fb Mon Sep 17 00:00:00 2001 From: Diip3sh Date: Tue, 13 Oct 2026 13:10:33 +0530 Subject: [PATCH 6/7] Fix three things the published CLI would have got wrong - dist/ had no baseline.json, so the built CLI crashed on start: the file was loaded with require, which tsc doesn't copy. It is a JSON import now, which tsc emits next to benchmark.js. - process.exit right after printing drops what a pipe hasn't taken yet, and pipes are asynchronous on macOS: zomb --json | jq (or an agent reading stdout) got the first 64 KB. The CLI flushes stdout first. - In a repo that doesn't gitignore node_modules, the scan read the installed packages as the repo's code, and a mention inside them kept a dead file alive. node_modules is never the repo now. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- src/benchmark.ts | 5 +++-- src/cli.ts | 30 ++++++++++++++++++------------ test/e2e.test.ts | 31 +++++++++++++++++++++++++++++++ tsconfig.json | 1 + 4 files changed, 53 insertions(+), 14 deletions(-) diff --git a/src/benchmark.ts b/src/benchmark.ts index 8410da9..53ef1d9 100644 --- a/src/benchmark.ts +++ b/src/benchmark.ts @@ -1,11 +1,12 @@ // How this repo compares with the 336 public repos measured in The State of AI Code. // Same arithmetic as the study, over the same window, so the two numbers are comparable. -import { createRequire } from 'node:module'; +// imported rather than required, so tsc copies the file into dist/ for the published CLI +import data from './baseline.json' with { type: 'json' }; import type { Month } from './types.ts'; type Author = { deletedPer100: number; medianCommit: number; addedShare: number }; type Baseline = { study: { repos: number; commits: number; lines: number; since: string; url: string }; authors: Record }; -const baseline: Baseline = createRequire(import.meta.url)('./baseline.json'); +const baseline: Baseline = data; export type Benchmark = { study: Baseline['study']; diff --git a/src/cli.ts b/src/cli.ts index e80e83d..cc34f8e 100755 --- a/src/cli.ts +++ b/src/cli.ts @@ -25,6 +25,11 @@ const paint = (tty) => ui({ width: process.stdout.columns || 80, color: Boolean( const { dim, red, yellow, green } = paint(process.stderr.isTTY); const log = (msg) => process.stderr.write(`${dim('zomb')} ${msg}\n`); const fail = (msg) => (process.stderr.write(`${red('✗')} ${msg}\n`), process.exit(1)); +// process.exit drops what a pipe hasn't taken yet, and pipes are asynchronous on macOS: `zomb --json | jq` got the first 64 KB +const exit = async (code: number): Promise => { + await new Promise((resolve) => process.stdout.write('', resolve)); + process.exit(code); +}; let args; try { @@ -52,7 +57,7 @@ const { values: opts, positionals } = args; if (opts.help) { const { version } = createRequire(import.meta.url)('../package.json'); console.log(renderHelp({ width: process.stdout.columns || 80, color: Boolean(process.stdout.isTTY) && !process.env.NO_COLOR, version })); - process.exit(0); + await exit(0); } // `zomb fix [path]` does the safe clean-up, `zomb blueprint [path]` proposes rules, `zomb show [path]` explains one finding; plain `zomb [path]` reports const command = ['fix', 'blueprint', 'show'].includes(positionals[0]) ? positionals.shift() : 'scan'; @@ -62,7 +67,7 @@ const LEVELS = { high: ['high'], medium: ['high', 'medium'], low: ['high', 'medi const exec = promisify(execFile); const sh = (cmd, args, cwd) => exec(cmd, args, { cwd, maxBuffer: 512 * 1024 * 1024 }).then((r) => r.stdout); const CODE = /\.[cm]?[jt]sx?$/; -const OWN = [':!.zomb', ':!*zomb-report*.html']; +const OWN = [':!.zomb', ':!*zomb-report*.html', ':(exclude,glob)**/node_modules/**']; // ignore everything in .zomb/ except the baseline and blueprint, which are meant to be committed const ZOMB_IGNORE = '*\n!baseline.json\n!blueprint.yml\n'; @@ -124,8 +129,9 @@ async function referencedBy(file) { return out.split('\n')[0] || null; } -// what's on disk: committed files plus new ones that aren't gitignored (uncommitted work is still your code) -const all = [...new Set((await sh('git', ['ls-files', '--cached', '--others', '--exclude-standard'], root)).split('\n').filter(Boolean))]; +// what's on disk: committed files plus new ones that aren't gitignored (uncommitted work is still your code); +// node_modules never is, even in a repo that forgot to ignore it +const all = [...new Set((await sh('git', ['ls-files', '--cached', '--others', '--exclude-standard'], root)).split('\n').filter((f) => f && !/(^|\/)node_modules\//.test(f)))]; const tracked = new Set((await sh('git', ['ls-files'], root)).split('\n').filter(Boolean)); const files = all.filter((f) => CODE.test(f) && !/\.d\.[cm]?ts$/.test(f)); if (!files.length) fail(`no JS/TS files tracked in ${root}`); @@ -254,7 +260,7 @@ if (command === 'show') { const t = tasks.find((x) => x.id === pick); if (!t) fail(tasks.length ? `there ${tasks.length === 1 ? 'is 1 finding' : `are ${tasks.length} findings`}: pick 1 to ${tasks.length}` : 'the scan found nothing to show'); else console.log(renderTask(t, tasks.length, { repo: data.repo, width: process.stdout.columns || 80, color: Boolean(process.stdout.isTTY) && !process.env.NO_COLOR })); - process.exit(0); + await exit(0); } if (command === 'blueprint') { @@ -264,14 +270,14 @@ if (command === 'blueprint') { const exists = existsSync(target); const color = Boolean(process.stdout.isTTY) && !process.env.NO_COLOR; console.log(renderBlueprint(proposal, today, { repo: data.repo, width: process.stdout.columns || 80, color, exists, write: opts.write })); - if (!opts.write) process.exit(0); + if (!opts.write) await exit(0); if (exists) fail(`${BLUEPRINT} already exists. Edit it, or delete it and run zomb blueprint --write again.`); await mkdir(path.dirname(target), { recursive: true }); await writeFile(target, toYaml(proposal, data.repo)); await writeFile(path.join(root, '.zomb', '.gitignore'), ZOMB_IGNORE); process.exit(0); } -if (command === 'fix') process.exit(await fix({ root, data, dryRun: opts['dry-run'], noChecks: opts['no-checks'], color: Boolean(process.stdout.isTTY) && !process.env.NO_COLOR, width: process.stdout.columns || 80 })); +if (command === 'fix') await exit(await fix({ root, data, dryRun: opts['dry-run'], noChecks: opts['no-checks'], color: Boolean(process.stdout.isTTY) && !process.env.NO_COLOR, width: process.stdout.columns || 80 })); // Baseline: the findings you already have. With one saved, only new findings fail --fail-on. const baselinePath = path.join(root, '.zomb', 'baseline.json'); @@ -284,9 +290,9 @@ if (opts['save-baseline']) { const baseline = existsSync(baselinePath) ? new Set(JSON.parse(await readFile(baselinePath, 'utf8')).fingerprints) : null; if (baseline) for (const t of tasks) t.new = !baseline.has(fingerprint(t)); const failing = opts['fail-on'] ? tasks.filter((t) => LEVELS[opts['fail-on']].includes(t.severity) && (!baseline || t.new)) : []; -const done = () => { +const done = async () => { if (failing.length) process.stderr.write(`${red('✗')} ${failing.length} ${baseline ? 'new ' : ''}finding${failing.length === 1 ? '' : 's'} at or above --fail-on ${opts['fail-on']}\n`); - process.exit(failing.length ? 1 : 0); + await exit(failing.length ? 1 : 0); }; // --json: the to-do list for an AI agent (or CI). No HTML, nothing else on stdout. @@ -304,12 +310,12 @@ if (opts.json) { ...(data.traffic && { traffic: data.traffic }), }; console.log(JSON.stringify({ repo: data.repo, commit: data.commit, since: data.since || null, baseline: Boolean(baseline), failing: failing.length, summary, tasks }, null, 2)); - done(); + await done(); } // --markdown: a PR comment or CI summary if (opts.markdown) { console.log(toMarkdown(tasks, { repo: data.repo, since: data.since, baseline })); - done(); + await done(); } // The report goes in .zomb/ (ignored by git) unless --out says otherwise; the baseline next to it is meant to be committed. @@ -334,4 +340,4 @@ const lines = render(data, tasks, { baseline, }); console.log(`\n${lines.join('\n')}\n`); -process.exit(failing.length ? 1 : 0); +await exit(failing.length ? 1 : 0); diff --git a/test/e2e.test.ts b/test/e2e.test.ts index a7a2833..e594113 100644 --- a/test/e2e.test.ts +++ b/test/e2e.test.ts @@ -78,6 +78,37 @@ test('zomb --traffic: routes no request reached, with or without links', () => { } }); +test('zomb --json: a big result arrives whole through a pipe, and an unignored node_modules is not the repo', () => { + const dir = mkdtempSync(path.join(tmpdir(), 'zomb-e2e-pipe-')); + try { + const app = { + 'package.json': JSON.stringify({ name: 'big', private: true }), + 'index.ts': 'export const main = 1;\n', + // no .gitignore: installed code sits next to the repo's own + 'node_modules/dep/package.json': JSON.stringify({ name: 'dep', main: 'index.js' }), + 'node_modules/dep/index.js': "module.exports = (s) => eval(s); // see lib/old0\n", + }; + for (let i = 0; i < 300; i++) app[`lib/old${i}.ts`] = `export const old${i} = ${i};\n`; + for (const [file, body] of Object.entries(app)) { + mkdirSync(path.dirname(path.join(dir, file)), { recursive: true }); + writeFileSync(path.join(dir, file), body); + } + const git = (...args) => execFileSync('git', args, { cwd: dir, stdio: 'ignore' }); + git('init', '-q'); + git('add', 'package.json', 'index.ts', 'lib'); + git('-c', 'user.name=Dev', '-c', 'user.email=dev@example.com', 'commit', '-q', '-m', 'init'); + + const out = execFileSync(process.execPath, [cli, dir, '--json'], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] }); + assert.ok(out.length > 64 * 1024, `the output is big enough to outgrow a pipe's buffer (${out.length} bytes)`); + const { tasks } = JSON.parse(out); + assert.equal(tasks.filter((t) => t.action === 'delete-file').length, 300); + assert.ok(!JSON.stringify(tasks).includes('node_modules'), 'nothing found in node_modules'); + assert.ok(tasks.some((t) => t.title.startsWith('Delete lib/old0.ts')), 'a mention inside node_modules does not keep a file alive'); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + test('zomb --json finds every planted mistake', () => { const dir = mkdtempSync(path.join(tmpdir(), 'zomb-e2e-')); try { diff --git a/tsconfig.json b/tsconfig.json index 0a4fc99..49aaee0 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -7,6 +7,7 @@ "strict": true, "noImplicitAny": false, "allowImportingTsExtensions": true, + "resolveJsonModule": true, "erasableSyntaxOnly": true, "verbatimModuleSyntax": true, "noEmit": true, From f8e275fd30fb8f81a67aa5f20a58a0ed135ce10e Mon Sep 17 00:00:00 2001 From: Diip3sh Date: Tue, 13 Oct 2026 13:10:33 +0530 Subject: [PATCH 7/7] Ready the launch: the study on Pages, 0.1.0, and a release workflow The Pages workflow now publishes site/ with The State of AI Code at /study/, and the CLI, the README, the landing page and the dashboard link there instead of the research folder. Version 0.1.0, with the repository field npm provenance needs. Pushing a v* tag tests and publishes to npm with trusted publishing, or an NPM_TOKEN secret for the first release. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- .claude-plugin/plugin.json | 2 +- .github/workflows/release.yml | 28 ++++++++++++++++++++++++++++ .github/workflows/site.yml | 19 ++++++++++++------- README.md | 2 +- package-lock.json | 4 ++-- package.json | 7 ++++++- site/index.html | 2 +- src/baseline.json | 2 +- 8 files changed, 52 insertions(+), 14 deletions(-) create mode 100644 .github/workflows/release.yml diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index a731c17..0c35c55 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "zomb", - "version": "0.0.1", + "version": "0.1.0", "description": "A health check for AI-written codebases. Guard checks every edit before it lands and stops keys, @ts-ignore, skipped tests, V2 copies and duplicate libraries; /zomb-guard and /zomb-clean check and clean the rest.", "author": { "name": "Open Dev Society" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..76f3ea5 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,28 @@ +# Publishes zomb to npm when a version tag (v0.1.0) is pushed, with provenance. +# Trusted publishing needs no token: on npmjs.com, add Open-Dev-Society/zomb and release.yml as the +# package's trusted publisher, with npm publish allowed. Before the package exists there, the first +# publish can use an NPM_TOKEN repo secret instead; npm tries trusted publishing first either way. +name: release +on: + push: + tags: ['v*'] +permissions: + contents: read + id-token: write +jobs: + publish: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-node@v7 + with: + node-version: 24 + registry-url: https://registry.npmjs.org + package-manager-cache: false + - name: The tag is the version in package.json + run: test "$GITHUB_REF_NAME" = "v$(node -p 'require("./package.json").version')" + - run: npm ci + - run: npm test + - run: npm publish --provenance --access public + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} diff --git a/.github/workflows/site.yml b/.github/workflows/site.yml index 3ae07fb..0a34307 100644 --- a/.github/workflows/site.yml +++ b/.github/workflows/site.yml @@ -1,11 +1,11 @@ -# Publishes site/ to GitHub Pages. Needs the repo public (or a paid plan), and Pages -# set to "GitHub Actions" in Settings once, so it is off until launch day. +# Publishes site/ to GitHub Pages, with The State of AI Code at /study/. Pages must be set to +# "GitHub Actions" in the repo's settings (once). name: site on: workflow_dispatch: push: branches: [main] - paths: ['site/**', '.github/workflows/site.yml'] + paths: ['site/**', 'research/data/state-of-ai-code.html', '.github/workflows/site.yml'] permissions: contents: read pages: write @@ -20,9 +20,14 @@ jobs: url: ${{ steps.deploy.outputs.page_url }} runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/upload-pages-artifact@v3 + - uses: actions/checkout@v7 + # the study page is rendered from research/data/summary.json by research/page.ts, and committed + - run: | + mkdir -p _site/study + cp -R site/. _site/ + cp research/data/state-of-ai-code.html _site/study/index.html + - uses: actions/upload-pages-artifact@v5 with: - path: site + path: _site - id: deploy - uses: actions/deploy-pages@v4 + uses: actions/deploy-pages@v5 diff --git a/README.md b/README.md index b736c10..f397178 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ zomb --traffic access.log # the routes no production request reached ## Compared with 336 public repos -Every whole-repo scan ends with how this repo compares against [The State of AI Code](research/): 52,688 commits in 336 public JS/TS repos since October 2025, measured the same way. +Every whole-repo scan ends with how this repo compares against [The State of AI Code](https://open-dev-society.github.io/zomb/study/): 52,688 commits in 336 public JS/TS repos since October 2025, measured the same way. ``` VS 336 PUBLIC REPOS ──────────────────────── agents wrote 100% of your new code diff --git a/package-lock.json b/package-lock.json index 0a3ec0c..e320e61 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "zomb", - "version": "0.0.1", + "version": "0.1.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "zomb", - "version": "0.0.1", + "version": "0.1.0", "license": "MIT", "dependencies": { "jscpd": "^5.3.2", diff --git a/package.json b/package.json index d5b614f..aa16276 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,12 @@ { "name": "zomb", - "version": "0.0.1", + "version": "0.1.0", "description": "A health check for AI-written codebases: security holes, zombie code, sprawl and architecture.", + "repository": { + "type": "git", + "url": "git+https://github.com/Open-Dev-Society/zomb.git" + }, + "homepage": "https://open-dev-society.github.io/zomb/", "type": "module", "bin": { "zomb": "dist/bin.js" diff --git a/site/index.html b/site/index.html index e0bfcc1..3cd2b96 100644 --- a/site/index.html +++ b/site/index.html @@ -463,7 +463,7 @@

Run it on the repo you are least sure about.

diff --git a/src/baseline.json b/src/baseline.json index 4adc75c..6a0c818 100644 --- a/src/baseline.json +++ b/src/baseline.json @@ -4,7 +4,7 @@ "commits": 52688, "lines": 18719425, "since": "2025-10-01", - "url": "https://github.com/Open-Dev-Society/zomb/tree/main/research" + "url": "https://open-dev-society.github.io/zomb/study/" }, "authors": { "Human": {