gstack/scripts/typecheck-test.ts

#!/usr/bin/env bun
/**
 * Test-code type-debt ratchet: `bun run typecheck:test [--write-baseline]`.
 *
 * Product code must typecheck clean (`bun run typecheck`). Test code carries
 * historical diagnostics, so it is held to a committed baseline instead:
 * each diagnostic identity (file + TS code + message, line-insensitive) maps
 * to how many times it occurs. The check fails on a new identity, on a higher
 * count, and on a stale baseline (fixed diagnostics must be locked in with
 * --write-baseline in the same change, so the allowance only ever shrinks).
 * Fixing one error and adding an identical-message error in the same file is
 * the one substitution this cannot see.
 */
import { spawnSync } from 'node:child_process';
import * as fs from 'node:fs';
import * as path from 'node:path';

export const BASELINE_FILE = 'scripts/typecheck-test-baseline.json';
const ROOT = path.resolve(import.meta.dir, '..');

export type DiagnosticCounts = Record<string, number>;

/** Parse `tsc --pretty false` output into identity → count. Continuation lines belong to the preceding diagnostic. */
export function parseDiagnostics(output: string): DiagnosticCounts {
  const counts: DiagnosticCounts = {};
  let current: string | null = null;
  const flush = () => {
    if (current !== null) counts[current] = (counts[current] ?? 0) + 1;
    current = null;
  };
  for (const line of output.split(/\r?\n/)) {
    const match = /^(.+?)\(\d+,\d+\): error (TS\d+): (.*)$/.exec(line);
    if (match) {
      flush();
      current = `${match[1]!.replaceAll('\\', '/')}\t${match[2]}\t${match[3]!.trim()}`;
    } else if (current !== null && /^\s+\S/.test(line)) {
      current += ` ${line.trim()}`;
    } else {
      flush();
    }
  }
  flush();
  return Object.fromEntries(Object.entries(counts).sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)));
}

export interface RatchetResult {
  added: Array<{ identity: string; baseline: number; current: number }>;
  fixed: Array<{ identity: string; baseline: number; current: number }>;
}

export function compareDiagnostics(baseline: DiagnosticCounts, current: DiagnosticCounts): RatchetResult {
  const added: RatchetResult['added'] = [], fixed: RatchetResult['fixed'] = [];
  for (const identity of new Set([...Object.keys(baseline), ...Object.keys(current)])) {
    const before = baseline[identity] ?? 0, now = current[identity] ?? 0;
    if (now > before) added.push({ identity, baseline: before, current: now });
    else if (now < before) fixed.push({ identity, baseline: before, current: now });
  }
  return { added, fixed };
}

export function readBaseline(file: string): DiagnosticCounts {
  let parsed: unknown;
  try {
    parsed = JSON.parse(fs.readFileSync(file, 'utf8'));
  } catch (error) {
    throw new Error(`test typecheck baseline ${file} is missing or unreadable (${(error as Error).message}). ` +
      'Regenerate it from a clean tree with: bun run typecheck:test --write-baseline');
  }
  const diagnostics = (parsed as { version?: unknown; diagnostics?: unknown })?.diagnostics;
  if ((parsed as { version?: unknown })?.version !== 1 || !diagnostics || typeof diagnostics !== 'object' ||
      !Object.values(diagnostics).every(n => Number.isSafeInteger(n) && (n as number) > 0)) {
    throw new Error(`test typecheck baseline ${file} is malformed. Regenerate it with: bun run typecheck:test --write-baseline`);
  }
  return diagnostics as DiagnosticCounts;
}

function describe(identity: string): string {
  const [file, code, message] = identity.split('\t');
  return `${file} ${code}: ${message}`;
}

function main(): number {
  const write = process.argv.includes('--write-baseline');
  const tsc = spawnSync(process.execPath, ['x', 'tsc', '-p', 'tsconfig.test.json', '--pretty', 'false'], {
    cwd: ROOT, encoding: 'utf8', timeout: 300_000, maxBuffer: 64 * 1024 * 1024,
  });
  if (tsc.error || tsc.signal) {
    console.error(`test typecheck could not run tsc: ${tsc.error?.message ?? tsc.signal}`);
    return 2;
  }
  const current = parseDiagnostics(`${tsc.stdout}\n${tsc.stderr}`);
  const total = Object.values(current).reduce((sum, n) => sum + n, 0);
  if (tsc.status !== 0 && total === 0) {
    console.error(`tsc exited ${tsc.status} without parseable diagnostics:\n${tsc.stdout}${tsc.stderr}`);
    return 2;
  }
  const baselinePath = path.join(ROOT, BASELINE_FILE);
  if (write) {
    fs.writeFileSync(baselinePath, JSON.stringify({ version: 1, diagnostics: current }, null, 2) + '\n');
    console.log(`test typecheck ratchet: wrote ${BASELINE_FILE} (${total} diagnostics, ${Object.keys(current).length} identities)`);
    return 0;
  }
  let baseline: DiagnosticCounts;
  try {
    baseline = readBaseline(baselinePath);
  } catch (error) {
    console.error((error as Error).message);
    return 1;
  }
  const { added, fixed } = compareDiagnostics(baseline, current);
  if (added.length) {
    console.error(`test typecheck ratchet: ${added.length} new or more frequent diagnostic(s). Fix them; do not add them to the baseline.`);
    for (const d of added) console.error(`  + ${describe(d.identity)} (${d.baseline} → ${d.current})`);
    console.error('Reproduce with: bunx tsc -p tsconfig.test.json --pretty false');
  }
  if (fixed.length) {
    console.error(`test typecheck ratchet: ${fixed.length} diagnostic(s) fixed. Lock the smaller allowance in with: bun run typecheck:test --write-baseline`);
    for (const d of fixed) console.error(`  - ${describe(d.identity)} (${d.baseline} → ${d.current})`);
  }
  if (added.length || fixed.length) return 1;
  console.log(`test typecheck ratchet: ${total} known diagnostics, none new.`);
  return 0;
}

if (import.meta.main) process.exit(main());