gstack/test/sealed-guard.test.ts

/**
 * Field wave CD4: the sealed-output guard. A repo that declares `sealed` in
 * `.gstack/pregate.json` cannot push sealed or eval output to a public
 * destination by accident: the managed pre-push hook and
 * `gstack-pregate sealed` enumerate every commit the push publishes (local
 * sha minus everything the destination remote already has), apply path
 * rules to every added, renamed or copied path (binary and empty files,
 * add-then-delete) and text rules to each commit's added lines. The policy
 * is the union of the base branch's and the pushed one; waivers come only
 * from the base branch's allow file; an unknown visibility counts as public.
 * Field incident: a per-cell transcript, a quoted sealed answer, a seed and a
 * /home/user path were pushed to a public eval repo with exit 0 and no output.
 */
import { afterEach, describe, expect, test } from 'bun:test';
import { spawnSync } from 'node:child_process';
import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import { resolveVisibility } from '../lib/sealed-guard';

const ROOT = path.resolve(import.meta.dir, '..');
const PREPUSH = path.join(ROOT, 'bin', 'gstack-redact-prepush');
const PREGATE = path.join(ROOT, 'bin', 'gstack-pregate');
const ZERO = '0'.repeat(40);
const SEALED = {
  paths: ['heldout/**'],
  transcripts: ['runs/**/transcript*.jsonl'],
  answer_markers: ['SEALED ANSWER:'],
  seed_patterns: ['shuffle_seed=\\d+'],
  local_paths: true,
};

interface Scratch { root: string; work: string; origin: string; state: string; env: Record<string, string> }
const scratches: Scratch[] = [];
afterEach(() => { while (scratches.length) fs.rmSync(scratches.pop()!.root, { recursive: true, force: true }); });

function sh(cwd: string, env: Record<string, string>, args: string[]): string {
  const r = spawnSync('git', args, { cwd, encoding: 'utf8', timeout: 30_000, env: { ...process.env, ...env } });
  if (r.status !== 0) throw new Error(`git ${args.join(' ')}: ${r.stderr}`);
  return (r.stdout ?? '').trim();
}

/** A bare origin with main carrying the given base files, and a clone on branch `results`. */
function scratch(baseFiles: Record<string, string | Buffer>): Scratch {
  const root = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'sealed-guard-')));
  const work = path.join(root, 'work'); const origin = path.join(root, 'origin.git'); const state = path.join(root, 'state');
  fs.mkdirSync(work); fs.mkdirSync(state);
  const env = { HOME: root, GSTACK_HOME: state, GSTACK_STATE_ROOT: state, GIT_CONFIG_GLOBAL: path.join(root, 'gitconfig'), GIT_CONFIG_NOSYSTEM: '1' };
  fs.writeFileSync(env.GIT_CONFIG_GLOBAL, '[user]\n\tname = t\n\temail = t@example.com\n[commit]\n\tgpgsign = false\n[init]\n\tdefaultBranch = main\n');
  spawnSync('git', ['init', '-q', '--bare', origin], { env: { ...process.env, ...env }, timeout: 30_000 });
  sh(work, env, ['init', '-q', '-b', 'main']);
  write(work, { 'README.md': '# evals\n', ...baseFiles });
  sh(work, env, ['add', '-A']); sh(work, env, ['commit', '-q', '-m', 'base']);
  sh(work, env, ['remote', 'add', 'origin', origin]);
  sh(work, env, ['push', '-q', 'origin', 'main']);
  sh(work, env, ['remote', 'set-head', 'origin', 'main']);
  sh(work, env, ['checkout', '-q', '-b', 'results']);
  const s = { root, work, origin, state, env };
  scratches.push(s);
  return s;
}
function write(work: string, files: Record<string, string | Buffer | null>): void {
  for (const [rel, body] of Object.entries(files)) {
    const abs = path.join(work, rel);
    if (body === null) { fs.rmSync(abs, { force: true }); continue; }
    fs.mkdirSync(path.dirname(abs), { recursive: true });
    fs.writeFileSync(abs, body);
  }
}
function commit(s: Scratch, files: Record<string, string | Buffer | null>, msg = 'results'): string {
  write(s.work, files);
  sh(s.work, s.env, ['add', '-A']); sh(s.work, s.env, ['commit', '-q', '-m', msg]);
  return sh(s.work, s.env, ['rev-parse', 'HEAD']);
}
const policy = (sealed: unknown) => JSON.stringify({ sealed }, null, 2) + '\n';

/** Drive the hook exactly as git does for `git push origin results` (a new remote branch). */
function hook(s: Scratch, extraEnv: Record<string, string> = {}) {
  const head = sh(s.work, s.env, ['rev-parse', 'HEAD']);
  const r = spawnSync('bun', [PREPUSH, 'origin', s.origin], { cwd: s.work, input: `refs/heads/results ${head} refs/heads/results ${ZERO}\n`, encoding: 'utf8', timeout: 60_000, env: { ...process.env, ...s.env, ...extraEnv } });
  return { code: r.status, err: r.stderr ?? '' };
}

describe('pre-push hook: public destination refuses each kind of sealed output', () => {
  test('a transcript, a sealed answer, a seed and a local home path each refuse, naming file, line and rule, never the matched text', () => {
    const cases: Array<[Record<string, string | Buffer>, string, RegExp]> = [
      [{ 'runs/cat40/cell-07/transcript.jsonl': '{"q":"x"}\n' }, 'sealed.transcript', /sealed\.transcript\s+runs\/cat40\/cell-07\/transcript\.jsonl/],
      [{ 'runs/cat40/summary.md': 'intro\nSEALED ANSWER: fund-a led the round\n' }, 'sealed.answer', /sealed\.answer\s+runs\/cat40\/summary\.md:2/],
      [{ 'runs/cat40/summary.md': 'run with shuffle_seed=42\n' }, 'sealed.seed', /sealed\.seed\s+runs\/cat40\/summary\.md:1/],
      [{ 'runs/cat40/summary.md': 'raw at /home/user/.capy/work/evals/raw.jsonl\n' }, 'sealed.local_path', /sealed\.local_path\s+runs\/cat40\/summary\.md:1/],
    ];
    for (const [files, rule, where] of cases) {
      const s = scratch({ '.gstack/pregate.json': policy(SEALED) });
      commit(s, files);
      const r = hook(s);
      expect(r.code).toBe(1);
      expect(r.err).toMatch(where);
      expect(r.err).toContain(rule);
      expect(r.err).toContain('(SEALED_OUTPUT_REFUSED)');
      expect(r.err).toContain('sealed.visibility');
      expect(r.err).not.toContain('fund-a led');
    }
  });

  test('path rules ignore content: a binary file, an empty transcript, an add-then-delete and a rename into a sealed path all refuse', () => {
    const binary = scratch({ '.gstack/pregate.json': policy(SEALED) });
    commit(binary, { 'heldout/questions.bin': Buffer.from([0, 1, 2, 0, 255, 0]) });
    expect(hook(binary).err).toMatch(/sealed\.path\s+heldout\/questions\.bin/);
    const empty = scratch({ '.gstack/pregate.json': policy(SEALED) });
    commit(empty, { 'runs/c1/transcript.jsonl': '' });
    expect(hook(empty).code).toBe(1);
    const gone = scratch({ '.gstack/pregate.json': policy(SEALED) });
    const added = commit(gone, { 'runs/c1/transcript.jsonl': '{"q":1}\n' });
    commit(gone, { 'runs/c1/transcript.jsonl': null }, 'remove it again');
    const g = hook(gone);
    expect(g.code).toBe(1);
    expect(g.err).toContain(`sealed.transcript  runs/c1/transcript.jsonl  commit ${added.slice(0, 12)}`);
    const moved = scratch({ '.gstack/pregate.json': policy(SEALED) });
    commit(moved, { 'scratch/t.jsonl': '{"q":1}\n{"q":2}\n{"q":3}\n' });
    fs.mkdirSync(path.join(moved.work, 'heldout'));
    sh(moved.work, moved.env, ['mv', 'scratch/t.jsonl', 'heldout/t.jsonl']);
    sh(moved.work, moved.env, ['commit', '-q', '-m', 'move']);
    expect(hook(moved).err).toMatch(/sealed\.path\s+heldout\/t\.jsonl/);
  });

  test('commits the destination already has are not this push: a branch from main that only adds a doc passes', () => {
    const s = scratch({ 'runs/old/transcript.jsonl': '{"q":"published before the policy"}\n', '.gstack/pregate.json': policy(SEALED) });
    commit(s, { 'docs/notes.md': 'notes\n' });
    const r = hook(s);
    expect(r.code).toBe(0);
    expect(r.err).not.toContain('sealed.');
  });
});

describe('policy union, waivers, private destinations and fail-closed parsing', () => {
  test('a push can tighten the policy but never relax it: dropping the base sealed config does not let a transcript through', () => {
    const s = scratch({ '.gstack/pregate.json': policy(SEALED) });
    commit(s, { '.gstack/pregate.json': '{}\n', 'runs/c1/transcript.jsonl': '{}\n' });
    expect(hook(s).code).toBe(1);
    const tighten = scratch({ '.gstack/pregate.json': policy(SEALED) });
    commit(tighten, { '.gstack/pregate.json': policy({ ...SEALED, paths: ['heldout/**', 'cells/**'] }), 'cells/c1.json': '{}\n' });
    expect(hook(tighten).err).toMatch(/sealed\.path\s+cells\/c1\.json/);
  });

  test('a waiver counts only from the base branch allow file and is listed; the same line committed in the push is ignored', () => {
    const s = scratch({ '.gstack/pregate.json': policy(SEALED) });
    const sha = commit(s, { 'runs/c1/transcript.jsonl': '{}\n' });
    const allow = `# rule\tpath\tsha\treason\nsealed.transcript\truns/c1/transcript.jsonl\t${sha.slice(0, 12)}\tsynthetic fixture, no sealed content\n`;
    commit(s, { '.gstack/sealed-allow.tsv': allow });
    expect(hook(s).code).toBe(1);
    sh(s.work, s.env, ['checkout', '-q', 'main']);
    write(s.work, { '.gstack/sealed-allow.tsv': allow });
    sh(s.work, s.env, ['add', '-A']); sh(s.work, s.env, ['commit', '-q', '-m', 'allow']); sh(s.work, s.env, ['push', '-q', 'origin', 'main']);
    sh(s.work, s.env, ['checkout', '-q', 'results']);
    const r = hook(s);
    expect(r.code).toBe(0);
    expect(r.err).toContain('waived sealed.transcript  runs/c1/transcript.jsonl');
  });

  test('a private destination (base override) warns and passes; a repo without the key sees no change; a malformed sealed policy refuses', () => {
    const priv = scratch({ '.gstack/pregate.json': policy({ ...SEALED, visibility: 'private' }) });
    commit(priv, { 'runs/c1/transcript.jsonl': '{}\n' });
    const p = hook(priv);
    expect(p.code).toBe(0);
    expect(p.err).toMatch(/warning.*sealed\.transcript\s+runs\/c1\/transcript\.jsonl/s);
    const none = scratch({});
    commit(none, { 'runs/c1/transcript.jsonl': '{}\n' });
    const n = hook(none);
    expect(n.code).toBe(0);
    expect(n.err).not.toContain('sealed');
    const bad = scratch({ '.gstack/pregate.json': policy(SEALED) });
    commit(bad, { '.gstack/pregate.json': policy({ paths: 'heldout/**' }), 'docs/x.md': 'x\n' });
    const b = hook(bad);
    expect(b.code).toBe(1);
    expect(b.err).toContain('(SEALED_OUTPUT_REFUSED)');
    expect(b.err).toContain('sealed policy');
  });
});

describe('gstack-pregate sealed (the standalone preflight before any push or PR creation)', () => {
  test('--range refuses the same findings with exit 1; the default range is what origin does not have yet', () => {
    const s = scratch({ '.gstack/pregate.json': policy(SEALED) });
    commit(s, { 'runs/c1/transcript.jsonl': '{}\n' });
    const run = (args: string[]) => spawnSync(PREGATE, ['sealed', ...args], { cwd: s.work, encoding: 'utf8', timeout: 60_000, env: { ...process.env, ...s.env } });
    const ranged = run(['--range', 'origin/main..HEAD', '--base', 'main']);
    expect(ranged.status).toBe(1);
    expect(ranged.stderr).toMatch(/sealed\.transcript\s+runs\/c1\/transcript\.jsonl/);
    expect(ranged.stdout).toMatch(/^SEALED: refused findings=1 commits=1 visibility=unknown \(counted public\)/m);
    const dflt = run(['--base', 'main']);
    expect(dflt.status).toBe(1);
    sh(s.work, s.env, ['push', '-q', 'origin', 'results']);
    const after = run(['--base', 'main']);
    expect(after.status).toBe(0);
    expect(after.stdout).toMatch(/^SEALED: clear findings=0 commits=0/m);
  });
});

describe('visibility (cache under the state root; unknown counts as public)', () => {
  test('a public answer is cached; a cached private answer older than an hour is re-queried before it can downgrade; a failed lookup is unknown', () => {
    const state = fs.mkdtempSync(path.join(os.tmpdir(), 'sealed-vis-'));
    try {
      let calls = 0;
      let answer: string | null = 'PUBLIC';
      const gh = () => { calls += 1; return answer === null ? { ok: false, out: '' } : { ok: true, out: JSON.stringify({ visibility: answer }) }; };
      const env = { GSTACK_STATE_ROOT: state };
      const t0 = Date.parse('2026-10-12T00:00:00Z');
      expect(resolveVisibility('https://github.com/acme-example/evals.git', { env, gh, now: t0 }).visibility).toBe('public');
      expect(resolveVisibility('https://github.com/acme-example/evals.git', { env, gh, now: t0 + 60_000 }).visibility).toBe('public');
      expect(calls).toBe(1);
      answer = 'PRIVATE';
      expect(resolveVisibility('git@github.com:acme-example/private-evals.git', { env, gh, now: t0 }).visibility).toBe('private');
      expect(resolveVisibility('git@github.com:acme-example/private-evals.git', { env, gh, now: t0 + 30 * 60_000 }).visibility).toBe('private');
      expect(calls).toBe(2);
      answer = null;
      const stale = resolveVisibility('git@github.com:acme-example/private-evals.git', { env, gh, now: t0 + 2 * 3600_000 });
      expect(calls).toBe(3);
      expect(stale.visibility).toBe('unknown');
      expect(resolveVisibility('/srv/git/evals.git', { env, gh, now: t0 }).visibility).toBe('unknown');
      expect(calls).toBe(3);
      expect(fs.existsSync(path.join(state, 'sealed-visibility.json'))).toBe(true);
    } finally { fs.rmSync(state, { recursive: true, force: true }); }
  });
});