/** * regen — the generated-file registry `.gstack/generated.json` (plan C4). * Every generated file a repo tracks is an entry: the output globs, the one * command that regenerates them and (optionally) the input globs that feed * it. Two verbs: `check` regenerates in a scratch `git worktree` seeded from * the working tree's content fingerprint (never a copy of a tree with * node_modules) and reports stale entries without touching the tree; `write` * regenerates in place. The /ship Step 3 conflict rule, gstack-restamp and * the pre-gate (lib/pregate/regen.ts) consume it. * * Trust boundary (plan C1): file presence never authorizes execution. The * registry is read from the committed copy on origin/ by default; the * caller opts in to running its commands with --allow-repo-commands or by * running under /ship. Containment: outputs and inputs are repo-relative * globs (lib/ship-policy.ts rules), commands are single lines under 512 * bytes, at most 64 entries. Docs: docs/pregate.md#generated. */ import { spawnSync } from 'node:child_process'; import { createHash } from 'node:crypto'; import * as fs from 'node:fs'; import * as os from 'node:os'; import * as path from 'node:path'; import { type ResultCodeName } from './result-codes'; import { runBashSync, wtreeFingerprint } from './script-invocation'; import { containedRelative, globToRegExp, matchesAny } from './ship-policy'; export const GENERATED_PATH = '.gstack/generated.json'; export const GENERATED_SCHEMA = 'https://github.com/garrytan/gstack/docs/pregate.md#generated-v1'; export interface RegenEntry { /** Stable id used in `regen` results and `--only `. */ id: string; /** The one command that regenerates every output; null for outputs regenerated by hand (listed, never run). */ command: string | null; /** Repo-relative globs of the files the command writes. */ outputs: string[]; /** Repo-relative globs of the files that feed the command; empty means "always regenerate". */ inputs: string[]; /** Free text shown in `list` and `--explain`. */ note?: string; } export interface Registry { $schema?: string; entries: RegenEntry[] } export interface RegistryError { code: ResultCodeName; path: string; message: string } const MAX_ENTRIES = 64; const MAX_COMMAND = 512; const ID_RE = /^[a-z0-9][a-z0-9_.-]{0,63}$/; export function validateRegistry(raw: unknown): RegistryError[] { const errors: RegistryError[] = []; if (!raw || typeof raw !== 'object' || Array.isArray(raw)) return [{ code: 'REGEN_REGISTRY_INVALID', path: '', message: 'registry must be a JSON object with an `entries` array' }]; const obj = raw as Record; for (const key of Object.keys(obj)) if (key !== '$schema' && key !== 'entries') errors.push({ code: 'REGEN_REGISTRY_INVALID', path: key, message: 'unknown key; valid keys: entries' }); if (!Array.isArray(obj.entries)) return [...errors, { code: 'REGEN_REGISTRY_INVALID', path: 'entries', message: 'must be an array' }]; if (obj.entries.length > MAX_ENTRIES) errors.push({ code: 'REGEN_REGISTRY_INVALID', path: 'entries', message: `more than ${MAX_ENTRIES} entries` }); const ids = new Set(); obj.entries.forEach((e: unknown, i: number) => { const at = `entries[${i}]`; if (!e || typeof e !== 'object' || Array.isArray(e)) { errors.push({ code: 'REGEN_REGISTRY_INVALID', path: at, message: 'must be an object' }); return; } const entry = e as Record; for (const key of Object.keys(entry)) if (!['id', 'command', 'outputs', 'inputs', 'note'].includes(key)) errors.push({ code: 'REGEN_REGISTRY_INVALID', path: `${at}.${key}`, message: 'unknown key; valid keys: id, command, outputs, inputs, note' }); if (typeof entry.id !== 'string' || !ID_RE.test(entry.id)) errors.push({ code: 'REGEN_REGISTRY_INVALID', path: `${at}.id`, message: 'must match [a-z0-9][a-z0-9_.-]{0,63}' }); else if (ids.has(entry.id)) errors.push({ code: 'REGEN_REGISTRY_INVALID', path: `${at}.id`, message: `duplicate id ${entry.id}` }); else ids.add(entry.id); if (entry.command !== null) { if (typeof entry.command !== 'string' || !entry.command.trim()) errors.push({ code: 'REGEN_REGISTRY_INVALID', path: `${at}.command`, message: 'must be a non-empty string or null' }); else if (entry.command.length > MAX_COMMAND || /[\0\r\n]/.test(entry.command)) errors.push({ code: 'REGEN_CONTAINMENT', path: `${at}.command`, message: `must be one line under ${MAX_COMMAND} characters` }); } for (const key of ['outputs', 'inputs'] as const) { const value = entry[key]; if (value === undefined && key === 'inputs') continue; if (!Array.isArray(value) || (key === 'outputs' && value.length === 0)) { errors.push({ code: 'REGEN_REGISTRY_INVALID', path: `${at}.${key}`, message: key === 'outputs' ? 'must be a non-empty array of globs' : 'must be an array of globs' }); continue; } if (value.length > MAX_ENTRIES) errors.push({ code: 'REGEN_CONTAINMENT', path: `${at}.${key}`, message: `more than ${MAX_ENTRIES} globs` }); value.forEach((g: unknown, j: number) => { if (typeof g !== 'string') { errors.push({ code: 'REGEN_REGISTRY_INVALID', path: `${at}.${key}[${j}]`, message: 'must be a string' }); return; } const why = containedRelative(g); if (why) { errors.push({ code: 'REGEN_CONTAINMENT', path: `${at}.${key}[${j}]`, message: `${JSON.stringify(g)} ${why}` }); return; } try { globToRegExp(g); } catch (err: any) { errors.push({ code: 'REGEN_CONTAINMENT', path: `${at}.${key}[${j}]`, message: err.message }); } }); } if (entry.note !== undefined && typeof entry.note !== 'string') errors.push({ code: 'REGEN_REGISTRY_INVALID', path: `${at}.note`, message: 'must be a string' }); }); return errors; } export function parseRegistry(text: string): { registry: Registry; errors: RegistryError[] } { let raw: unknown; try { raw = JSON.parse(text); } catch (e: any) { return { registry: { entries: [] }, errors: [{ code: 'REGEN_REGISTRY_INVALID', path: GENERATED_PATH, message: `not valid JSON: ${e.message}` }] }; } const errors = validateRegistry(raw); if (errors.length) return { registry: { entries: [] }, errors }; const obj = raw as { $schema?: string; entries: Array> }; return { registry: { $schema: obj.$schema, entries: obj.entries.map(e => ({ id: e.id!, command: e.command ?? null, outputs: e.outputs!, inputs: e.inputs ?? [], note: e.note })) }, errors: [] }; } function git(cwd: string, args: string[]): { ok: boolean; out: string; err: string } { const r = spawnSync('git', args, { cwd, encoding: 'utf8', timeout: 60_000, maxBuffer: 64 * 1024 * 1024 }); return { ok: r.status === 0, out: (r.stdout ?? '').trim(), err: (r.stderr ?? '').trim() }; } export interface RegistrySource { kind: 'base' | 'head' | 'none'; label: string; sha: string | null } export type RegistryLoad = { registry: Registry; errors: RegistryError[]; present: boolean; source: RegistrySource } | { unavailable: true; base: string; detail: string }; /** The reviewed registry: `.gstack/generated.json` as committed on origin/; unresolvable base is `unavailable` (never a silent fallback). */ export function loadRegistryFromBase(repoRoot: string, base: string): RegistryLoad { const ref = `origin/${base.replace(/^origin\//, '')}`; const sha = git(repoRoot, ['rev-parse', '--verify', '--quiet', `${ref}^{commit}`]); if (!sha.ok || !sha.out) return { unavailable: true, base: base.replace(/^origin\//, ''), detail: `${ref} is not a resolvable commit` }; const show = git(repoRoot, ['show', `${sha.out}:${GENERATED_PATH}`]); if (!show.ok) return { registry: { entries: [] }, errors: [], present: false, source: { kind: 'none', label: `${ref}@${sha.out.slice(0, 12)} (no ${GENERATED_PATH})`, sha: sha.out } }; return { ...parseRegistry(show.out), present: true, source: { kind: 'base', label: `${ref}@${sha.out.slice(0, 12)}:${GENERATED_PATH}`, sha: sha.out } }; } /** The working-tree registry: unreviewed, labeled so in every record. */ export function loadRegistryFromHead(repoRoot: string): RegistryLoad { const file = path.join(repoRoot, GENERATED_PATH); if (!fs.existsSync(file)) return { registry: { entries: [] }, errors: [], present: false, source: { kind: 'none', label: `head (no ${GENERATED_PATH})`, sha: null } }; return { ...parseRegistry(fs.readFileSync(file, 'utf8')), present: true, source: { kind: 'head', label: `head (unreviewed) ${GENERATED_PATH}`, sha: null } }; } export function isRegistryUnavailable(load: RegistryLoad): load is { unavailable: true; base: string; detail: string } { return (load as any).unavailable === true; } /** Entries whose inputs or outputs intersect `touched`; every entry when `touched` is undefined or an entry declares no inputs. */ export function selectEntries(registry: Registry, touched?: readonly string[]): RegenEntry[] { if (!touched) return registry.entries; return registry.entries.filter(e => e.inputs.length === 0 || touched.some(t => matchesAny(e.inputs, t) || matchesAny(e.outputs, t))); } export interface Scratch { dir: string; wtree: string; remove: () => void } /** * A detached `git worktree` seeded from the working tree's content * fingerprint, with the repository's ignored node_modules linked in (a * symlink, never a copy). The caller removes it; a failure to create it is an * Error naming the step. */ export function scratchWorktree(repoRoot: string, prefix = 'gstack-regen-'): Scratch { const fp = wtreeFingerprint({ cwd: repoRoot }); if ('error' in fp) throw new Error(`cannot fingerprint the working tree (${fp.error})`); const wtree = fp.tree; const dir = fs.mkdtempSync(path.join(os.tmpdir(), prefix)); fs.rmSync(dir, { recursive: true, force: true }); const add = git(repoRoot, ['worktree', 'add', '--detach', '-q', dir, 'HEAD']); if (!add.ok) throw new Error(`cannot create the scratch worktree: ${add.err}`); const remove = () => { git(repoRoot, ['worktree', 'remove', '--force', dir]); fs.rmSync(dir, { recursive: true, force: true }); git(repoRoot, ['worktree', 'prune']); }; const seed = git(dir, ['read-tree', '--reset', '-u', wtree]); if (!seed.ok) { remove(); throw new Error(`cannot seed the scratch worktree: ${seed.err}`); } const nm = path.join(repoRoot, 'node_modules'); if (fs.existsSync(nm) && !fs.existsSync(path.join(dir, 'node_modules')) && git(repoRoot, ['check-ignore', '-q', 'node_modules']).ok) { try { fs.symlinkSync(nm, path.join(dir, 'node_modules'), 'dir'); } catch { /* the command runs without deps */ } } return { dir, wtree, remove }; } export interface EntryResult { id: string; command: string | null; status: 'fresh' | 'stale' | 'failed' | 'listed' | 'written' | 'skipped' | 'deps-missing'; /** Output files whose bytes changed (stale) or were written. */ changed: string[]; /** Files the command wrote outside its declared outputs (a refusal in `check`). */ outside: string[]; exit?: number | null; stderr?: string; ms: number; /** The exact command an author runs to refresh the entry. */ fix: string; /** `check` with `prove`, on a fresh entry: the deleted outputs the proof run recreated byte-identically (an output it never writes, or a proof run that failed, proves nothing). */ proven?: string[]; } /** Working tree vs the seeded index in a scratch worktree: modified tracked files plus new untracked ones. */ function changedPaths(dir: string): string[] { const modified = git(dir, ['diff', '--name-only', '-z']).out.split('\0'); const deleted = git(dir, ['ls-files', '--deleted', '-z']).out.split('\0'); const untracked = git(dir, ['ls-files', '--others', '--exclude-standard', '-z']).out.split('\0'); return [...new Set([...modified, ...deleted, ...untracked])].filter(p => p && p !== 'node_modules').sort(); } /** sha256 per tracked and untracked non-ignored file (write mode compares two of these). */ function snapshotFiles(root: string): Map { const out = new Map(); for (const rel of git(root, ['ls-files', '-co', '--exclude-standard', '-z']).out.split('\0').filter(Boolean)) { try { out.set(rel, createHash('sha256').update(fs.readFileSync(path.join(root, rel))).digest('hex')); } catch { out.set(rel, 'missing'); } } return out; } function diffSnapshots(a: Map, b: Map): string[] { const changed = new Set(); for (const [k, v] of b) if (a.get(k) !== v) changed.add(k); for (const k of a.keys()) if (!b.has(k)) changed.add(k); return [...changed].filter(p => p !== 'node_modules').sort(); } /** * The install command for a repository whose package.json declares * dependencies but has no node_modules (the scratch worktree links none * then), picked from its lockfile; null when nothing is missing. */ export function missingDependencies(repoRoot: string): string | null { if (fs.existsSync(path.join(repoRoot, 'node_modules'))) return null; let pkg: { dependencies?: Record; devDependencies?: Record }; try { pkg = JSON.parse(fs.readFileSync(path.join(repoRoot, 'package.json'), 'utf8')) ?? {}; } catch { return null; } if (!Object.keys(pkg.dependencies ?? {}).length && !Object.keys(pkg.devDependencies ?? {}).length) return null; const lockfiles: Array<[string, string]> = [['bun.lock', 'bun install'], ['bun.lockb', 'bun install'], ['pnpm-lock.yaml', 'pnpm install'], ['yarn.lock', 'yarn install'], ['package-lock.json', 'npm install']]; return lockfiles.find(([file]) => fs.existsSync(path.join(repoRoot, file)))?.[1] ?? 'bun install'; } function runCommand(cwd: string, command: string, env: NodeJS.ProcessEnv, timeoutMs: number): { exit: number | null; stderr: string } { const r = runBashSync(`set -o pipefail; ${command}`, { cwd, encoding: 'utf8', timeout: timeoutMs, maxBuffer: 64 * 1024 * 1024, env: { ...env, GSTACK_REGEN_SCRATCH: cwd } }); return { exit: r.status, stderr: ((r.stderr ?? '') + (r.error ? `\n${r.error.message}` : '')).trim().slice(-4000) }; } export interface RunRegenOptions { repoRoot: string; registry: Registry; mode: 'check' | 'write'; /** Restrict to these entry ids. */ only?: string[]; /** Restrict to entries whose inputs or outputs intersect these paths. */ touched?: string[]; /** `check`: outputs to prove on a fresh entry: a second run with each one deleted first, the ones it recreates byte-identically come back as `proven`. The verdict comes from the first run alone. */ prove?: readonly string[]; env?: NodeJS.ProcessEnv; timeoutMs?: number; } /** * `check`: run every selected command in one scratch worktree and compare its * outputs with the working tree; `write`: run them in the repository. An * entry with `command: null` is `listed` (its fix line names the outputs an * author refreshes by hand). A `check` whose outputs differ while the * repository's dependencies are not installed is `deps-missing`, not `stale`: * the generator may have run without them. Never runs a command the caller did not opt into: * the bin enforces --allow-repo-commands before calling this. */ export function runRegen(o: RunRegenOptions): { results: EntryResult[]; wtree: string | null } { const entries = selectEntries(o.registry, o.touched).filter(e => !o.only || o.only.includes(e.id)); const timeout = o.timeoutMs ?? 600_000; const results: EntryResult[] = []; let scratch: Scratch | null = null; const install = o.mode === 'check' ? missingDependencies(o.repoRoot) : null; const fixFor = (e: RegenEntry) => e.command ? `${e.command} # then commit ${e.outputs.join(' ')}` : `regenerate by hand: ${e.outputs.join(' ')}${e.note ? ` (${e.note})` : ''}`; try { if (o.mode === 'check' && entries.some(e => e.command)) scratch = scratchWorktree(o.repoRoot); for (const e of entries) { const started = Date.now(); if (!e.command) { results.push({ id: e.id, command: null, status: 'listed', changed: [], outside: [], ms: 0, fix: fixFor(e) }); continue; } const cwd = o.mode === 'check' ? scratch!.dir : o.repoRoot; const before = o.mode === 'write' ? snapshotFiles(o.repoRoot) : null; const run = runCommand(cwd, e.command, o.env ?? process.env, timeout); const after = before ? diffSnapshots(before, snapshotFiles(o.repoRoot)) : changedPaths(cwd); const changed = after.filter(p => matchesAny(e.outputs, p)); const outside = after.filter(p => !matchesAny(e.outputs, p) && !matchesAny(o.registry.entries.flatMap(x => x.outputs), p)); const ms = Date.now() - started; if (run.exit !== 0) { results.push({ id: e.id, command: e.command, status: 'failed', changed, outside, exit: run.exit, stderr: run.stderr, ms, fix: fixFor(e) }); continue; } if (o.mode === 'check') { const differs = changed.length > 0 || outside.length > 0; // Reset the scratch so the next entry's diff is its own. const reset = () => { git(cwd, ['checkout', '-q', '--', '.']); git(cwd, ['clean', '-fdq']); }; reset(); if (differs && install) { results.push({ id: e.id, command: e.command, status: 'deps-missing', changed, outside, exit: 0, ms, fix: `run ${install}, then rerun the check` }); continue; } // The proof is a second run apart from the verdict: each touched output deleted first, the ones recreated // byte-identically are proven, and a failure (a generator that reads its own output) proves nothing. const probe = differs ? [] : (o.prove ?? []).filter(p => matchesAny(e.outputs, p) && fs.existsSync(path.join(cwd, p))); let proven: string[] | undefined; if (probe.length) { for (const p of probe) fs.rmSync(path.join(cwd, p), { force: true }); const proof = runCommand(cwd, e.command, o.env ?? process.env, timeout); const redo = changedPaths(cwd); proven = proof.exit === 0 ? probe.filter(p => !redo.includes(p)) : []; reset(); } results.push({ id: e.id, command: e.command, status: differs ? 'stale' : 'fresh', changed, outside, exit: 0, ms, fix: fixFor(e), ...(proven ? { proven } : {}) }); } else results.push({ id: e.id, command: e.command, status: changed.length ? 'written' : 'fresh', changed, outside, exit: 0, ms, fix: fixFor(e) }); } } finally { scratch?.remove(); } if (scratch) return { results, wtree: scratch.wtree }; const fp = wtreeFingerprint({ cwd: o.repoRoot }); return { results, wtree: 'tree' in fp ? fp.tree : null }; } /** One line per entry, the way `gstack-regen` and the pre-gate print them. */ export function renderEntry(r: EntryResult): string { const head = `regen ${r.id}: ${r.status}${r.ms ? ` (${r.ms} ms)` : ''}`; if (r.status === 'fresh' || r.status === 'skipped') return head; if (r.status === 'listed') return `${head} — ${r.fix}`; if (r.status === 'failed') return `${head} — exit ${r.exit}: ${(r.stderr ?? '').split('\n').pop() ?? ''}; fix: run \`${r.command}\` and read its output (REGEN_COMMAND_FAILED)`; if (r.status === 'deps-missing') return `${head} — dependencies missing (no node_modules); ${r.fix} — differs without them: ${[...r.changed, ...r.outside].join(' ')} (REGEN_DEPS_MISSING)`; const parts = [head]; if (r.changed.length) parts.push(`stale: ${r.changed.join(' ')}`); if (r.outside.length) parts.push(`outside declared outputs: ${r.outside.join(' ')}`); parts.push(`fix: ${r.fix}${r.status === 'stale' ? ' (REGEN_STALE)' : ''}`); return parts.join(' — '); } /** The example `init` writes. */ export function exampleRegistryJson(): string { const example: Registry = { $schema: GENERATED_SCHEMA, entries: [ { id: 'example', command: 'npm run generate', outputs: ['generated/**'], inputs: ['src/schema/**'], note: 'replace with the repo’s own generator; command: null lists outputs regenerated by hand' }, ], }; return JSON.stringify(example, null, 2) + '\n'; }