gstack/lib/regen.ts

/**
 * regen — the generated-file registry `.gstack/generated.json` (plan C4).
 * Every generated file a repo tracks is an entry: the output globs, the one
 * command that regenerates them and (optionally) the input globs that feed
 * it. Two verbs: `check` regenerates in a scratch `git worktree` seeded from
 * the working tree's content fingerprint (never a copy of a tree with
 * node_modules) and reports stale entries without touching the tree; `write`
 * regenerates in place. The /ship Step 3 conflict rule, gstack-restamp and
 * the pre-gate (lib/pregate/regen.ts) consume it.
 *
 * Trust boundary (plan C1): file presence never authorizes execution. The
 * registry is read from the committed copy on origin/<base> by default; the
 * caller opts in to running its commands with --allow-repo-commands or by
 * running under /ship. Containment: outputs and inputs are repo-relative
 * globs (lib/ship-policy.ts rules), commands are single lines under 512
 * bytes, at most 64 entries. Docs: docs/pregate.md#generated.
 */
import { spawnSync } from 'node:child_process';
import { createHash } from 'node:crypto';
import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import { type ResultCodeName } from './result-codes';
import { runBashSync, wtreeFingerprint } from './script-invocation';
import { containedRelative, globToRegExp, matchesAny } from './ship-policy';

export const GENERATED_PATH = '.gstack/generated.json';
export const GENERATED_SCHEMA = 'https://github.com/garrytan/gstack/docs/pregate.md#generated-v1';

export interface RegenEntry {
  /** Stable id used in `regen` results and `--only <id>`. */
  id: string;
  /** The one command that regenerates every output; null for outputs regenerated by hand (listed, never run). */
  command: string | null;
  /** Repo-relative globs of the files the command writes. */
  outputs: string[];
  /** Repo-relative globs of the files that feed the command; empty means "always regenerate". */
  inputs: string[];
  /** Free text shown in `list` and `--explain`. */
  note?: string;
}
export interface Registry { $schema?: string; entries: RegenEntry[] }
export interface RegistryError { code: ResultCodeName; path: string; message: string }

const MAX_ENTRIES = 64;
const MAX_COMMAND = 512;
const ID_RE = /^[a-z0-9][a-z0-9_.-]{0,63}$/;

export function validateRegistry(raw: unknown): RegistryError[] {
  const errors: RegistryError[] = [];
  if (!raw || typeof raw !== 'object' || Array.isArray(raw)) return [{ code: 'REGEN_REGISTRY_INVALID', path: '', message: 'registry must be a JSON object with an `entries` array' }];
  const obj = raw as Record<string, unknown>;
  for (const key of Object.keys(obj)) if (key !== '$schema' && key !== 'entries') errors.push({ code: 'REGEN_REGISTRY_INVALID', path: key, message: 'unknown key; valid keys: entries' });
  if (!Array.isArray(obj.entries)) return [...errors, { code: 'REGEN_REGISTRY_INVALID', path: 'entries', message: 'must be an array' }];
  if (obj.entries.length > MAX_ENTRIES) errors.push({ code: 'REGEN_REGISTRY_INVALID', path: 'entries', message: `more than ${MAX_ENTRIES} entries` });
  const ids = new Set<string>();
  obj.entries.forEach((e: unknown, i: number) => {
    const at = `entries[${i}]`;
    if (!e || typeof e !== 'object' || Array.isArray(e)) { errors.push({ code: 'REGEN_REGISTRY_INVALID', path: at, message: 'must be an object' }); return; }
    const entry = e as Record<string, unknown>;
    for (const key of Object.keys(entry)) if (!['id', 'command', 'outputs', 'inputs', 'note'].includes(key)) errors.push({ code: 'REGEN_REGISTRY_INVALID', path: `${at}.${key}`, message: 'unknown key; valid keys: id, command, outputs, inputs, note' });
    if (typeof entry.id !== 'string' || !ID_RE.test(entry.id)) errors.push({ code: 'REGEN_REGISTRY_INVALID', path: `${at}.id`, message: 'must match [a-z0-9][a-z0-9_.-]{0,63}' });
    else if (ids.has(entry.id)) errors.push({ code: 'REGEN_REGISTRY_INVALID', path: `${at}.id`, message: `duplicate id ${entry.id}` });
    else ids.add(entry.id);
    if (entry.command !== null) {
      if (typeof entry.command !== 'string' || !entry.command.trim()) errors.push({ code: 'REGEN_REGISTRY_INVALID', path: `${at}.command`, message: 'must be a non-empty string or null' });
      else if (entry.command.length > MAX_COMMAND || /[\0\r\n]/.test(entry.command)) errors.push({ code: 'REGEN_CONTAINMENT', path: `${at}.command`, message: `must be one line under ${MAX_COMMAND} characters` });
    }
    for (const key of ['outputs', 'inputs'] as const) {
      const value = entry[key];
      if (value === undefined && key === 'inputs') continue;
      if (!Array.isArray(value) || (key === 'outputs' && value.length === 0)) { errors.push({ code: 'REGEN_REGISTRY_INVALID', path: `${at}.${key}`, message: key === 'outputs' ? 'must be a non-empty array of globs' : 'must be an array of globs' }); continue; }
      if (value.length > MAX_ENTRIES) errors.push({ code: 'REGEN_CONTAINMENT', path: `${at}.${key}`, message: `more than ${MAX_ENTRIES} globs` });
      value.forEach((g: unknown, j: number) => {
        if (typeof g !== 'string') { errors.push({ code: 'REGEN_REGISTRY_INVALID', path: `${at}.${key}[${j}]`, message: 'must be a string' }); return; }
        const why = containedRelative(g);
        if (why) { errors.push({ code: 'REGEN_CONTAINMENT', path: `${at}.${key}[${j}]`, message: `${JSON.stringify(g)} ${why}` }); return; }
        try { globToRegExp(g); } catch (err: any) { errors.push({ code: 'REGEN_CONTAINMENT', path: `${at}.${key}[${j}]`, message: err.message }); }
      });
    }
    if (entry.note !== undefined && typeof entry.note !== 'string') errors.push({ code: 'REGEN_REGISTRY_INVALID', path: `${at}.note`, message: 'must be a string' });
  });
  return errors;
}

export function parseRegistry(text: string): { registry: Registry; errors: RegistryError[] } {
  let raw: unknown;
  try { raw = JSON.parse(text); }
  catch (e: any) { return { registry: { entries: [] }, errors: [{ code: 'REGEN_REGISTRY_INVALID', path: GENERATED_PATH, message: `not valid JSON: ${e.message}` }] }; }
  const errors = validateRegistry(raw);
  if (errors.length) return { registry: { entries: [] }, errors };
  const obj = raw as { $schema?: string; entries: Array<Partial<RegenEntry>> };
  return { registry: { $schema: obj.$schema, entries: obj.entries.map(e => ({ id: e.id!, command: e.command ?? null, outputs: e.outputs!, inputs: e.inputs ?? [], note: e.note })) }, errors: [] };
}

function git(cwd: string, args: string[]): { ok: boolean; out: string; err: string } {
  const r = spawnSync('git', args, { cwd, encoding: 'utf8', timeout: 60_000, maxBuffer: 64 * 1024 * 1024 });
  return { ok: r.status === 0, out: (r.stdout ?? '').trim(), err: (r.stderr ?? '').trim() };
}

export interface RegistrySource { kind: 'base' | 'head' | 'none'; label: string; sha: string | null }
export type RegistryLoad = { registry: Registry; errors: RegistryError[]; present: boolean; source: RegistrySource } | { unavailable: true; base: string; detail: string };

/** The reviewed registry: `.gstack/generated.json` as committed on origin/<base>; unresolvable base is `unavailable` (never a silent fallback). */
export function loadRegistryFromBase(repoRoot: string, base: string): RegistryLoad {
  const ref = `origin/${base.replace(/^origin\//, '')}`;
  const sha = git(repoRoot, ['rev-parse', '--verify', '--quiet', `${ref}^{commit}`]);
  if (!sha.ok || !sha.out) return { unavailable: true, base: base.replace(/^origin\//, ''), detail: `${ref} is not a resolvable commit` };
  const show = git(repoRoot, ['show', `${sha.out}:${GENERATED_PATH}`]);
  if (!show.ok) return { registry: { entries: [] }, errors: [], present: false, source: { kind: 'none', label: `${ref}@${sha.out.slice(0, 12)} (no ${GENERATED_PATH})`, sha: sha.out } };
  return { ...parseRegistry(show.out), present: true, source: { kind: 'base', label: `${ref}@${sha.out.slice(0, 12)}:${GENERATED_PATH}`, sha: sha.out } };
}

/** The working-tree registry: unreviewed, labeled so in every record. */
export function loadRegistryFromHead(repoRoot: string): RegistryLoad {
  const file = path.join(repoRoot, GENERATED_PATH);
  if (!fs.existsSync(file)) return { registry: { entries: [] }, errors: [], present: false, source: { kind: 'none', label: `head (no ${GENERATED_PATH})`, sha: null } };
  return { ...parseRegistry(fs.readFileSync(file, 'utf8')), present: true, source: { kind: 'head', label: `head (unreviewed) ${GENERATED_PATH}`, sha: null } };
}

export function isRegistryUnavailable(load: RegistryLoad): load is { unavailable: true; base: string; detail: string } {
  return (load as any).unavailable === true;
}

/** Entries whose inputs or outputs intersect `touched`; every entry when `touched` is undefined or an entry declares no inputs. */
export function selectEntries(registry: Registry, touched?: readonly string[]): RegenEntry[] {
  if (!touched) return registry.entries;
  return registry.entries.filter(e => e.inputs.length === 0 || touched.some(t => matchesAny(e.inputs, t) || matchesAny(e.outputs, t)));
}

export interface Scratch { dir: string; wtree: string; remove: () => void }

/**
 * A detached `git worktree` seeded from the working tree's content
 * fingerprint, with the repository's ignored node_modules linked in (a
 * symlink, never a copy). The caller removes it; a failure to create it is an
 * Error naming the step.
 */
export function scratchWorktree(repoRoot: string, prefix = 'gstack-regen-'): Scratch {
  const fp = wtreeFingerprint({ cwd: repoRoot });
  if ('error' in fp) throw new Error(`cannot fingerprint the working tree (${fp.error})`);
  const wtree = fp.tree;
  const dir = fs.mkdtempSync(path.join(os.tmpdir(), prefix));
  fs.rmSync(dir, { recursive: true, force: true });
  const add = git(repoRoot, ['worktree', 'add', '--detach', '-q', dir, 'HEAD']);
  if (!add.ok) throw new Error(`cannot create the scratch worktree: ${add.err}`);
  const remove = () => { git(repoRoot, ['worktree', 'remove', '--force', dir]); fs.rmSync(dir, { recursive: true, force: true }); git(repoRoot, ['worktree', 'prune']); };
  const seed = git(dir, ['read-tree', '--reset', '-u', wtree]);
  if (!seed.ok) { remove(); throw new Error(`cannot seed the scratch worktree: ${seed.err}`); }
  const nm = path.join(repoRoot, 'node_modules');
  if (fs.existsSync(nm) && !fs.existsSync(path.join(dir, 'node_modules')) && git(repoRoot, ['check-ignore', '-q', 'node_modules']).ok) {
    try { fs.symlinkSync(nm, path.join(dir, 'node_modules'), 'dir'); } catch { /* the command runs without deps */ }
  }
  return { dir, wtree, remove };
}

export interface EntryResult {
  id: string; command: string | null;
  status: 'fresh' | 'stale' | 'failed' | 'listed' | 'written' | 'skipped' | 'deps-missing';
  /** Output files whose bytes changed (stale) or were written. */
  changed: string[];
  /** Files the command wrote outside its declared outputs (a refusal in `check`). */
  outside: string[];
  exit?: number | null; stderr?: string; ms: number;
  /** The exact command an author runs to refresh the entry. */
  fix: string;
  /** `check` with `prove`, on a fresh entry: the deleted outputs the proof run recreated byte-identically (an output it never writes, or a proof run that failed, proves nothing). */
  proven?: string[];
}

/** Working tree vs the seeded index in a scratch worktree: modified tracked files plus new untracked ones. */
function changedPaths(dir: string): string[] {
  const modified = git(dir, ['diff', '--name-only', '-z']).out.split('\0');
  const deleted = git(dir, ['ls-files', '--deleted', '-z']).out.split('\0');
  const untracked = git(dir, ['ls-files', '--others', '--exclude-standard', '-z']).out.split('\0');
  return [...new Set([...modified, ...deleted, ...untracked])].filter(p => p && p !== 'node_modules').sort();
}

/** sha256 per tracked and untracked non-ignored file (write mode compares two of these). */
function snapshotFiles(root: string): Map<string, string> {
  const out = new Map<string, string>();
  for (const rel of git(root, ['ls-files', '-co', '--exclude-standard', '-z']).out.split('\0').filter(Boolean)) {
    try { out.set(rel, createHash('sha256').update(fs.readFileSync(path.join(root, rel))).digest('hex')); } catch { out.set(rel, 'missing'); }
  }
  return out;
}
function diffSnapshots(a: Map<string, string>, b: Map<string, string>): string[] {
  const changed = new Set<string>();
  for (const [k, v] of b) if (a.get(k) !== v) changed.add(k);
  for (const k of a.keys()) if (!b.has(k)) changed.add(k);
  return [...changed].filter(p => p !== 'node_modules').sort();
}

/**
 * The install command for a repository whose package.json declares
 * dependencies but has no node_modules (the scratch worktree links none
 * then), picked from its lockfile; null when nothing is missing.
 */
export function missingDependencies(repoRoot: string): string | null {
  if (fs.existsSync(path.join(repoRoot, 'node_modules'))) return null;
  let pkg: { dependencies?: Record<string, unknown>; devDependencies?: Record<string, unknown> };
  try { pkg = JSON.parse(fs.readFileSync(path.join(repoRoot, 'package.json'), 'utf8')) ?? {}; } catch { return null; }
  if (!Object.keys(pkg.dependencies ?? {}).length && !Object.keys(pkg.devDependencies ?? {}).length) return null;
  const lockfiles: Array<[string, string]> = [['bun.lock', 'bun install'], ['bun.lockb', 'bun install'], ['pnpm-lock.yaml', 'pnpm install'], ['yarn.lock', 'yarn install'], ['package-lock.json', 'npm install']];
  return lockfiles.find(([file]) => fs.existsSync(path.join(repoRoot, file)))?.[1] ?? 'bun install';
}

function runCommand(cwd: string, command: string, env: NodeJS.ProcessEnv, timeoutMs: number): { exit: number | null; stderr: string } {
  const r = runBashSync(`set -o pipefail; ${command}`, { cwd, encoding: 'utf8', timeout: timeoutMs, maxBuffer: 64 * 1024 * 1024, env: { ...env, GSTACK_REGEN_SCRATCH: cwd } });
  return { exit: r.status, stderr: ((r.stderr ?? '') + (r.error ? `\n${r.error.message}` : '')).trim().slice(-4000) };
}

export interface RunRegenOptions {
  repoRoot: string; registry: Registry; mode: 'check' | 'write';
  /** Restrict to these entry ids. */
  only?: string[];
  /** Restrict to entries whose inputs or outputs intersect these paths. */
  touched?: string[];
  /** `check`: outputs to prove on a fresh entry: a second run with each one deleted first, the ones it recreates byte-identically come back as `proven`. The verdict comes from the first run alone. */
  prove?: readonly string[];
  env?: NodeJS.ProcessEnv; timeoutMs?: number;
}

/**
 * `check`: run every selected command in one scratch worktree and compare its
 * outputs with the working tree; `write`: run them in the repository. An
 * entry with `command: null` is `listed` (its fix line names the outputs an
 * author refreshes by hand). A `check` whose outputs differ while the
 * repository's dependencies are not installed is `deps-missing`, not `stale`:
 * the generator may have run without them. Never runs a command the caller did not opt into:
 * the bin enforces --allow-repo-commands before calling this.
 */
export function runRegen(o: RunRegenOptions): { results: EntryResult[]; wtree: string | null } {
  const entries = selectEntries(o.registry, o.touched).filter(e => !o.only || o.only.includes(e.id));
  const timeout = o.timeoutMs ?? 600_000;
  const results: EntryResult[] = [];
  let scratch: Scratch | null = null;
  const install = o.mode === 'check' ? missingDependencies(o.repoRoot) : null;
  const fixFor = (e: RegenEntry) => e.command ? `${e.command}  # then commit ${e.outputs.join(' ')}` : `regenerate by hand: ${e.outputs.join(' ')}${e.note ? ` (${e.note})` : ''}`;
  try {
    if (o.mode === 'check' && entries.some(e => e.command)) scratch = scratchWorktree(o.repoRoot);
    for (const e of entries) {
      const started = Date.now();
      if (!e.command) { results.push({ id: e.id, command: null, status: 'listed', changed: [], outside: [], ms: 0, fix: fixFor(e) }); continue; }
      const cwd = o.mode === 'check' ? scratch!.dir : o.repoRoot;
      const before = o.mode === 'write' ? snapshotFiles(o.repoRoot) : null;
      const run = runCommand(cwd, e.command, o.env ?? process.env, timeout);
      const after = before ? diffSnapshots(before, snapshotFiles(o.repoRoot)) : changedPaths(cwd);
      const changed = after.filter(p => matchesAny(e.outputs, p));
      const outside = after.filter(p => !matchesAny(e.outputs, p) && !matchesAny(o.registry.entries.flatMap(x => x.outputs), p));
      const ms = Date.now() - started;
      if (run.exit !== 0) { results.push({ id: e.id, command: e.command, status: 'failed', changed, outside, exit: run.exit, stderr: run.stderr, ms, fix: fixFor(e) }); continue; }
      if (o.mode === 'check') {
        const differs = changed.length > 0 || outside.length > 0;
        // Reset the scratch so the next entry's diff is its own.
        const reset = () => { git(cwd, ['checkout', '-q', '--', '.']); git(cwd, ['clean', '-fdq']); };
        reset();
        if (differs && install) { results.push({ id: e.id, command: e.command, status: 'deps-missing', changed, outside, exit: 0, ms, fix: `run ${install}, then rerun the check` }); continue; }
        // The proof is a second run apart from the verdict: each touched output deleted first, the ones recreated
        // byte-identically are proven, and a failure (a generator that reads its own output) proves nothing.
        const probe = differs ? [] : (o.prove ?? []).filter(p => matchesAny(e.outputs, p) && fs.existsSync(path.join(cwd, p)));
        let proven: string[] | undefined;
        if (probe.length) {
          for (const p of probe) fs.rmSync(path.join(cwd, p), { force: true });
          const proof = runCommand(cwd, e.command, o.env ?? process.env, timeout);
          const redo = changedPaths(cwd);
          proven = proof.exit === 0 ? probe.filter(p => !redo.includes(p)) : [];
          reset();
        }
        results.push({ id: e.id, command: e.command, status: differs ? 'stale' : 'fresh', changed, outside, exit: 0, ms, fix: fixFor(e), ...(proven ? { proven } : {}) });
      } else results.push({ id: e.id, command: e.command, status: changed.length ? 'written' : 'fresh', changed, outside, exit: 0, ms, fix: fixFor(e) });
    }
  } finally { scratch?.remove(); }
  if (scratch) return { results, wtree: scratch.wtree };
  const fp = wtreeFingerprint({ cwd: o.repoRoot });
  return { results, wtree: 'tree' in fp ? fp.tree : null };
}

/** One line per entry, the way `gstack-regen` and the pre-gate print them. */
export function renderEntry(r: EntryResult): string {
  const head = `regen ${r.id}: ${r.status}${r.ms ? ` (${r.ms} ms)` : ''}`;
  if (r.status === 'fresh' || r.status === 'skipped') return head;
  if (r.status === 'listed') return `${head} — ${r.fix}`;
  if (r.status === 'failed') return `${head} — exit ${r.exit}: ${(r.stderr ?? '').split('\n').pop() ?? ''}; fix: run \`${r.command}\` and read its output (REGEN_COMMAND_FAILED)`;
  if (r.status === 'deps-missing') return `${head} — dependencies missing (no node_modules); ${r.fix} — differs without them: ${[...r.changed, ...r.outside].join(' ')} (REGEN_DEPS_MISSING)`;
  const parts = [head];
  if (r.changed.length) parts.push(`stale: ${r.changed.join(' ')}`);
  if (r.outside.length) parts.push(`outside declared outputs: ${r.outside.join(' ')}`);
  parts.push(`fix: ${r.fix}${r.status === 'stale' ? ' (REGEN_STALE)' : ''}`);
  return parts.join(' — ');
}

/** The example `init` writes. */
export function exampleRegistryJson(): string {
  const example: Registry = {
    $schema: GENERATED_SCHEMA,
    entries: [
      { id: 'example', command: 'npm run generate', outputs: ['generated/**'], inputs: ['src/schema/**'], note: 'replace with the repo’s own generator; command: null lists outputs regenerated by hand' },
    ],
  };
  return JSON.stringify(example, null, 2) + '\n';
}