/** * Field wave CD4: the sealed-output guard. A repo that declares `sealed` in * `.gstack/pregate.json` cannot push sealed or eval output to a public * destination by accident: the managed pre-push hook and * `gstack-pregate sealed` enumerate every commit the push publishes (local * sha minus everything the destination remote already has), apply path * rules to every added, renamed or copied path (binary and empty files, * add-then-delete) and text rules to each commit's added lines. The policy * is the union of the base branch's and the pushed one; waivers come only * from the base branch's allow file; an unknown visibility counts as public. * Field incident: a per-cell transcript, a quoted sealed answer, a seed and a * /home/user path were pushed to a public eval repo with exit 0 and no output. */ import { afterEach, describe, expect, test } from 'bun:test'; import { spawnSync } from 'node:child_process'; import * as fs from 'node:fs'; import * as os from 'node:os'; import * as path from 'node:path'; import { resolveVisibility } from '../lib/sealed-guard'; const ROOT = path.resolve(import.meta.dir, '..'); const PREPUSH = path.join(ROOT, 'bin', 'gstack-redact-prepush'); const PREGATE = path.join(ROOT, 'bin', 'gstack-pregate'); const ZERO = '0'.repeat(40); const SEALED = { paths: ['heldout/**'], transcripts: ['runs/**/transcript*.jsonl'], answer_markers: ['SEALED ANSWER:'], seed_patterns: ['shuffle_seed=\\d+'], local_paths: true, }; interface Scratch { root: string; work: string; origin: string; state: string; env: Record } const scratches: Scratch[] = []; afterEach(() => { while (scratches.length) fs.rmSync(scratches.pop()!.root, { recursive: true, force: true }); }); function sh(cwd: string, env: Record, args: string[]): string { const r = spawnSync('git', args, { cwd, encoding: 'utf8', timeout: 30_000, env: { ...process.env, ...env } }); if (r.status !== 0) throw new Error(`git ${args.join(' ')}: ${r.stderr}`); return (r.stdout ?? '').trim(); } /** A bare origin with main carrying the given base files, and a clone on branch `results`. */ function scratch(baseFiles: Record): Scratch { const root = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'sealed-guard-'))); const work = path.join(root, 'work'); const origin = path.join(root, 'origin.git'); const state = path.join(root, 'state'); fs.mkdirSync(work); fs.mkdirSync(state); const env = { HOME: root, GSTACK_HOME: state, GSTACK_STATE_ROOT: state, GIT_CONFIG_GLOBAL: path.join(root, 'gitconfig'), GIT_CONFIG_NOSYSTEM: '1' }; fs.writeFileSync(env.GIT_CONFIG_GLOBAL, '[user]\n\tname = t\n\temail = t@example.com\n[commit]\n\tgpgsign = false\n[init]\n\tdefaultBranch = main\n'); spawnSync('git', ['init', '-q', '--bare', origin], { env: { ...process.env, ...env }, timeout: 30_000 }); sh(work, env, ['init', '-q', '-b', 'main']); write(work, { 'README.md': '# evals\n', ...baseFiles }); sh(work, env, ['add', '-A']); sh(work, env, ['commit', '-q', '-m', 'base']); sh(work, env, ['remote', 'add', 'origin', origin]); sh(work, env, ['push', '-q', 'origin', 'main']); sh(work, env, ['remote', 'set-head', 'origin', 'main']); sh(work, env, ['checkout', '-q', '-b', 'results']); const s = { root, work, origin, state, env }; scratches.push(s); return s; } function write(work: string, files: Record): void { for (const [rel, body] of Object.entries(files)) { const abs = path.join(work, rel); if (body === null) { fs.rmSync(abs, { force: true }); continue; } fs.mkdirSync(path.dirname(abs), { recursive: true }); fs.writeFileSync(abs, body); } } function commit(s: Scratch, files: Record, msg = 'results'): string { write(s.work, files); sh(s.work, s.env, ['add', '-A']); sh(s.work, s.env, ['commit', '-q', '-m', msg]); return sh(s.work, s.env, ['rev-parse', 'HEAD']); } const policy = (sealed: unknown) => JSON.stringify({ sealed }, null, 2) + '\n'; /** Drive the hook exactly as git does for `git push origin results` (a new remote branch). */ function hook(s: Scratch, extraEnv: Record = {}) { const head = sh(s.work, s.env, ['rev-parse', 'HEAD']); const r = spawnSync('bun', [PREPUSH, 'origin', s.origin], { cwd: s.work, input: `refs/heads/results ${head} refs/heads/results ${ZERO}\n`, encoding: 'utf8', timeout: 60_000, env: { ...process.env, ...s.env, ...extraEnv } }); return { code: r.status, err: r.stderr ?? '' }; } describe('pre-push hook: public destination refuses each kind of sealed output', () => { test('a transcript, a sealed answer, a seed and a local home path each refuse, naming file, line and rule, never the matched text', () => { const cases: Array<[Record, string, RegExp]> = [ [{ 'runs/cat40/cell-07/transcript.jsonl': '{"q":"x"}\n' }, 'sealed.transcript', /sealed\.transcript\s+runs\/cat40\/cell-07\/transcript\.jsonl/], [{ 'runs/cat40/summary.md': 'intro\nSEALED ANSWER: fund-a led the round\n' }, 'sealed.answer', /sealed\.answer\s+runs\/cat40\/summary\.md:2/], [{ 'runs/cat40/summary.md': 'run with shuffle_seed=42\n' }, 'sealed.seed', /sealed\.seed\s+runs\/cat40\/summary\.md:1/], [{ 'runs/cat40/summary.md': 'raw at /home/user/.capy/work/evals/raw.jsonl\n' }, 'sealed.local_path', /sealed\.local_path\s+runs\/cat40\/summary\.md:1/], ]; for (const [files, rule, where] of cases) { const s = scratch({ '.gstack/pregate.json': policy(SEALED) }); commit(s, files); const r = hook(s); expect(r.code).toBe(1); expect(r.err).toMatch(where); expect(r.err).toContain(rule); expect(r.err).toContain('(SEALED_OUTPUT_REFUSED)'); expect(r.err).toContain('sealed.visibility'); expect(r.err).not.toContain('fund-a led'); } }); test('path rules ignore content: a binary file, an empty transcript, an add-then-delete and a rename into a sealed path all refuse', () => { const binary = scratch({ '.gstack/pregate.json': policy(SEALED) }); commit(binary, { 'heldout/questions.bin': Buffer.from([0, 1, 2, 0, 255, 0]) }); expect(hook(binary).err).toMatch(/sealed\.path\s+heldout\/questions\.bin/); const empty = scratch({ '.gstack/pregate.json': policy(SEALED) }); commit(empty, { 'runs/c1/transcript.jsonl': '' }); expect(hook(empty).code).toBe(1); const gone = scratch({ '.gstack/pregate.json': policy(SEALED) }); const added = commit(gone, { 'runs/c1/transcript.jsonl': '{"q":1}\n' }); commit(gone, { 'runs/c1/transcript.jsonl': null }, 'remove it again'); const g = hook(gone); expect(g.code).toBe(1); expect(g.err).toContain(`sealed.transcript runs/c1/transcript.jsonl commit ${added.slice(0, 12)}`); const moved = scratch({ '.gstack/pregate.json': policy(SEALED) }); commit(moved, { 'scratch/t.jsonl': '{"q":1}\n{"q":2}\n{"q":3}\n' }); fs.mkdirSync(path.join(moved.work, 'heldout')); sh(moved.work, moved.env, ['mv', 'scratch/t.jsonl', 'heldout/t.jsonl']); sh(moved.work, moved.env, ['commit', '-q', '-m', 'move']); expect(hook(moved).err).toMatch(/sealed\.path\s+heldout\/t\.jsonl/); }); test('commits the destination already has are not this push: a branch from main that only adds a doc passes', () => { const s = scratch({ 'runs/old/transcript.jsonl': '{"q":"published before the policy"}\n', '.gstack/pregate.json': policy(SEALED) }); commit(s, { 'docs/notes.md': 'notes\n' }); const r = hook(s); expect(r.code).toBe(0); expect(r.err).not.toContain('sealed.'); }); }); describe('policy union, waivers, private destinations and fail-closed parsing', () => { test('a push can tighten the policy but never relax it: dropping the base sealed config does not let a transcript through', () => { const s = scratch({ '.gstack/pregate.json': policy(SEALED) }); commit(s, { '.gstack/pregate.json': '{}\n', 'runs/c1/transcript.jsonl': '{}\n' }); expect(hook(s).code).toBe(1); const tighten = scratch({ '.gstack/pregate.json': policy(SEALED) }); commit(tighten, { '.gstack/pregate.json': policy({ ...SEALED, paths: ['heldout/**', 'cells/**'] }), 'cells/c1.json': '{}\n' }); expect(hook(tighten).err).toMatch(/sealed\.path\s+cells\/c1\.json/); }); test('a waiver counts only from the base branch allow file and is listed; the same line committed in the push is ignored', () => { const s = scratch({ '.gstack/pregate.json': policy(SEALED) }); const sha = commit(s, { 'runs/c1/transcript.jsonl': '{}\n' }); const allow = `# rule\tpath\tsha\treason\nsealed.transcript\truns/c1/transcript.jsonl\t${sha.slice(0, 12)}\tsynthetic fixture, no sealed content\n`; commit(s, { '.gstack/sealed-allow.tsv': allow }); expect(hook(s).code).toBe(1); sh(s.work, s.env, ['checkout', '-q', 'main']); write(s.work, { '.gstack/sealed-allow.tsv': allow }); sh(s.work, s.env, ['add', '-A']); sh(s.work, s.env, ['commit', '-q', '-m', 'allow']); sh(s.work, s.env, ['push', '-q', 'origin', 'main']); sh(s.work, s.env, ['checkout', '-q', 'results']); const r = hook(s); expect(r.code).toBe(0); expect(r.err).toContain('waived sealed.transcript runs/c1/transcript.jsonl'); }); test('a private destination (base override) warns and passes; a repo without the key sees no change; a malformed sealed policy refuses', () => { const priv = scratch({ '.gstack/pregate.json': policy({ ...SEALED, visibility: 'private' }) }); commit(priv, { 'runs/c1/transcript.jsonl': '{}\n' }); const p = hook(priv); expect(p.code).toBe(0); expect(p.err).toMatch(/warning.*sealed\.transcript\s+runs\/c1\/transcript\.jsonl/s); const none = scratch({}); commit(none, { 'runs/c1/transcript.jsonl': '{}\n' }); const n = hook(none); expect(n.code).toBe(0); expect(n.err).not.toContain('sealed'); const bad = scratch({ '.gstack/pregate.json': policy(SEALED) }); commit(bad, { '.gstack/pregate.json': policy({ paths: 'heldout/**' }), 'docs/x.md': 'x\n' }); const b = hook(bad); expect(b.code).toBe(1); expect(b.err).toContain('(SEALED_OUTPUT_REFUSED)'); expect(b.err).toContain('sealed policy'); }); }); describe('gstack-pregate sealed (the standalone preflight before any push or PR creation)', () => { test('--range refuses the same findings with exit 1; the default range is what origin does not have yet', () => { const s = scratch({ '.gstack/pregate.json': policy(SEALED) }); commit(s, { 'runs/c1/transcript.jsonl': '{}\n' }); const run = (args: string[]) => spawnSync(PREGATE, ['sealed', ...args], { cwd: s.work, encoding: 'utf8', timeout: 60_000, env: { ...process.env, ...s.env } }); const ranged = run(['--range', 'origin/main..HEAD', '--base', 'main']); expect(ranged.status).toBe(1); expect(ranged.stderr).toMatch(/sealed\.transcript\s+runs\/c1\/transcript\.jsonl/); expect(ranged.stdout).toMatch(/^SEALED: refused findings=1 commits=1 visibility=unknown \(counted public\)/m); const dflt = run(['--base', 'main']); expect(dflt.status).toBe(1); sh(s.work, s.env, ['push', '-q', 'origin', 'results']); const after = run(['--base', 'main']); expect(after.status).toBe(0); expect(after.stdout).toMatch(/^SEALED: clear findings=0 commits=0/m); }); }); describe('visibility (cache under the state root; unknown counts as public)', () => { test('a public answer is cached; a cached private answer older than an hour is re-queried before it can downgrade; a failed lookup is unknown', () => { const state = fs.mkdtempSync(path.join(os.tmpdir(), 'sealed-vis-')); try { let calls = 0; let answer: string | null = 'PUBLIC'; const gh = () => { calls += 1; return answer === null ? { ok: false, out: '' } : { ok: true, out: JSON.stringify({ visibility: answer }) }; }; const env = { GSTACK_STATE_ROOT: state }; const t0 = Date.parse('2026-10-12T00:00:00Z'); expect(resolveVisibility('https://github.com/acme-example/evals.git', { env, gh, now: t0 }).visibility).toBe('public'); expect(resolveVisibility('https://github.com/acme-example/evals.git', { env, gh, now: t0 + 60_000 }).visibility).toBe('public'); expect(calls).toBe(1); answer = 'PRIVATE'; expect(resolveVisibility('git@github.com:acme-example/private-evals.git', { env, gh, now: t0 }).visibility).toBe('private'); expect(resolveVisibility('git@github.com:acme-example/private-evals.git', { env, gh, now: t0 + 30 * 60_000 }).visibility).toBe('private'); expect(calls).toBe(2); answer = null; const stale = resolveVisibility('git@github.com:acme-example/private-evals.git', { env, gh, now: t0 + 2 * 3600_000 }); expect(calls).toBe(3); expect(stale.visibility).toBe('unknown'); expect(resolveVisibility('/srv/git/evals.git', { env, gh, now: t0 }).visibility).toBe('unknown'); expect(calls).toBe(3); expect(fs.existsSync(path.join(state, 'sealed-visibility.json'))).toBe(true); } finally { fs.rmSync(state, { recursive: true, force: true }); } }); });