From f254cebf1200c20cd48af4d2d44fd032c0192aad Mon Sep 17 00:00:00 2001 From: garrytan Date: Mon, 2 Nov 2026 03:13:34 +0000 Subject: [PATCH] test(review): planner misreads: SVG chart as runtime, session data as auth, bare base in diff-scope and receipt depth (GST-911, GST-884, failing first) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- test/review-planner-misreads.test.ts | 236 +++++++++++++++++++++++++++ 1 file changed, 236 insertions(+) create mode 100644 test/review-planner-misreads.test.ts diff --git a/test/review-planner-misreads.test.ts b/test/review-planner-misreads.test.ts new file mode 100644 index 000000000..f85f066ae --- /dev/null +++ b/test/review-planner-misreads.test.ts @@ -0,0 +1,236 @@ +/** + * Three /ship planner misreads from GST-911 (gstack-evals#132, a results write-up) and GST-884: + * a plotting-script SVG counted as runtime code, a data file named for sessions read as auth code, and + * gstack-diff-scope (plus the receipt depth) resolving a bare base name, so a stale local main or a tag + * named main or origin/main moved the diff. Every real signal stays: an SVG that can run code is + * security, session code is auth, and a credential reference is a secret wherever it is. + */ +import { afterEach, describe, expect, test } from 'bun:test'; +import { spawnSync } from 'node:child_process'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { collectCommit, collectLive } from '../lib/review-collect'; +import { depthOfCommitInput } from '../lib/review-depth'; +import { targetedClasses } from '../lib/review-targeted'; +import { receiptDepthAtHead, receiptDepthVerdict } from '../lib/ship-receipt-depth'; + +const BIN = path.join(import.meta.dir, '..', 'bin'); +const SCOPE = path.join(BIN, 'gstack-diff-scope'); +const dirs: string[] = []; +afterEach(() => { while (dirs.length) fs.rmSync(dirs.pop()!, { recursive: true, force: true }); }); + +function fixture() { + const root = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'planner-misreads-'))); + dirs.push(root); + const work = path.join(root, 'work'); + const env = { ...process.env, GSTACK_STATE_ROOT: path.join(root, 'state'), GIT_AUTHOR_NAME: 't', GIT_AUTHOR_EMAIL: 't@t', GIT_COMMITTER_NAME: 't', GIT_COMMITTER_EMAIL: 't@t', GIT_CONFIG_GLOBAL: path.join(root, 'gitconfig'), GIT_CONFIG_NOSYSTEM: '1' }; + fs.writeFileSync(path.join(root, 'gitconfig'), '[commit]\n\tgpgsign = false\n[init]\n\tdefaultBranch = main\n'); + const sh = (cwd: string, c: string) => { const r = spawnSync('bash', ['-c', c], { cwd, env, encoding: 'utf8', timeout: 60_000 }); if (r.status !== 0) throw new Error(`${c}: ${r.stderr}`); return r.stdout.trim(); }; + sh(root, 'git init -q --bare origin.git && git clone -q origin.git work 2>/dev/null'); + const write = (files: Record) => { for (const [p, text] of Object.entries(files)) { const f = path.join(work, p); fs.mkdirSync(path.dirname(f), { recursive: true }); fs.writeFileSync(f, text); } }; + const commit = (files: Record) => { write(files); sh(work, 'git add -A && git commit -qm change'); return sh(work, 'git rev-parse HEAD'); }; + write({ 'README.md': '# evals\n', 'src/run.ts': 'export const run = () => 1;\n' }); + sh(work, 'git add -A && git commit -qm base && git push -q origin main && git checkout -qb feat'); + const base = sh(work, 'git rev-parse HEAD'); + const scope = (args: string[]) => { + const r = spawnSync('bash', [SCOPE, ...args], { cwd: work, env, encoding: 'utf8', timeout: 30_000 }); + return { status: r.status, out: r.stdout, vars: Object.fromEntries(r.stdout.split('\n').filter(l => /^SCOPE_\w+=/.test(l)).map(l => l.split('=') as [string, string])) }; + }; + return { root, work, env, sh, write, commit, base, scope }; +} + +/** A matplotlib-style chart: XML declaration, DOCTYPE, RDF metadata, CSS, markers by internal reference, a clip path. */ +const chart = (points: number, extra = '') => [ + '', + '', + '', + ' ', + ' ', + ' ', + ' ', + ' 2026-10-31T12:00:00', + ' image/svg+xml', + ' ', + ' ', + ' Matplotlib v3.9.2, https://matplotlib.org/', + ' ', + ' ', + ' ', + ' ', + ' ', + ' ', + ' ', + ' ', + ' ', + ' ', + ' ', + ' ', + ' ', + ...Array.from({ length: points }, (_, i) => ` `), + ' ', + ' cost per run & pass rate', + ' ', + ...(extra ? [extra] : []), + ' ', + ' ', + ' ', + ' ', + ' ', + ' ', + '', + '', +].join('\n'); + +const plan = (fx: ReturnType, head: string) => { + const c = collectCommit(fx.work, fx.base, head); + const d = depthOfCommitInput({ ...c, gateTagged: 0 }); + return { class: d.class, depth: d.depth, runtime: d.runtimeLines, risk: d.risk }; +}; + +describe('GST-911 misread 1: a chart SVG is data, an SVG that can run code is security', () => { + // Value: protects=a results write-up whose chart is an SVG plans like the same chart as a PNG (docs, light, no runtime lines); + // fails_when=an SVG with only drawing elements, internal references and CSS counts as runtime code; seam=collectCommit + depthOfCommitInput, collectLive + test('a plotting-script SVG no code names plans as docs, light, runtime 0, like the PNG', () => { + const fx = fixture(); + const svg = plan(fx, fx.commit({ 'results/2026-10-31/chart.svg': chart(600), 'results/2026-10-31/README.md': '# Results\n\n![chart](chart.svg)\n' })); + expect(svg).toEqual({ class: 'docs', depth: 'light', runtime: 0, risk: [] }); + fx.sh(fx.work, 'git rm -q results/2026-10-31/chart.svg && git commit -qm drop'); + const png = plan(fx, fx.commit({ 'results/2026-10-31/chart.png': '\x89PNG\r\n\x1a\n\0\0\0\rIHDR' })); + expect(png).toEqual({ class: 'docs', depth: 'light', runtime: 0, risk: [] }); + fx.write({ 'results/live/chart.svg': chart(40) }); + const live = collectLive(fx.work, 'main'); + expect(depthOfCommitInput({ ...live, gateTagged: 0 }).runtimeLines).toBe(0); + }, 60_000); + + // Value: protects=an SVG that can execute script stays code and gets the security class (full depth), however it spells the script; + // fails_when=a script element, a namespaced script, an event handler, a javascript: URL (plain or entity-encoded) or a + // foreignObject plans as docs or loses the security class; seam=collectCommit + depthOfCommitInput + test.each([ + ['a script element', ' '], + ['a namespaced script', ' window.x = 42'], + ['an event handler', ' '], + ['a javascript: URL', ' x'], + ['an entity-encoded javascript: URL', ' x'], + ['a foreignObject', '
x
'], + ])('%s stays code with the security class', (_name, line) => { + const fx = fixture(); + expect(plan(fx, fx.commit({ 'results/chart.svg': chart(5, line) }))).toMatchObject({ class: 'code', depth: 'full', risk: ['security'] }); + }, 60_000); + + // Value: protects=only an SVG made of drawing elements is data: anything that loads, links or declares outside that set stays code; + // fails_when=an external image or stylesheet, an entity declaration, an animation or a CSS url() to another document reads as docs; seam=collectCommit + depthOfCommitInput + test.each([ + ['an external image', ' '], + ['a CSS import', ' '], + ['a CSS url() to another document', ' '], + ['an animation', ' '], + ['a stylesheet processing instruction', ''], + ['an entity declaration', ' ]>'], + ])('%s keeps the SVG code', (_name, line) => { + const fx = fixture(); + expect(plan(fx, fx.commit({ 'results/chart.svg': chart(5, line) })).class).toBe('code'); + }, 60_000); + + // Value: protects=a passive edit to an SVG that already carries a script is still code, because the whole file decides, not the + // changed lines; and an SVG code names stays code like any consumed data file; fails_when=either plans as docs; seam=collectCommit + test('an edit inside a scripted SVG, and an SVG code names, stay code', () => { + const fx = fixture(); + fx.commit({ 'assets/live.svg': chart(5, ' ') }); + fx.sh(fx.work, 'git checkout -q main && git merge -q --ff-only feat && git checkout -q feat'); + const scripted = { ...fx, base: fx.sh(fx.work, 'git rev-parse HEAD') }; + expect(plan(scripted, fx.commit({ 'assets/live.svg': chart(6, ' ') })).class).toBe('code'); + const named = plan(fx, fx.commit({ 'assets/icon.svg': chart(3), 'src/icon.ts': "export const icon = new URL('./icon.svg', import.meta.url);\n" })); + expect(named.class).toBe('code'); + expect(named.runtime).toBeGreaterThan(3); + }, 60_000); +}); + +describe('GST-911 misread 2: a data file named for sessions is not auth', () => { + // Value: protects=a data file whose name says session never sets SCOPE_AUTH, while session code and session config still do; + // fails_when=recovered-sessions.jsonl sets SCOPE_AUTH, or session.ts, a session store, a sessions controller or config/sessions.json stop setting it; + // seam=bin/gstack-diff-scope + test('gstack-diff-scope: session data files are not auth; session code and config are', () => { + for (const [file, auth] of [ + ['results/recovered-sessions.jsonl', 'false'], ['data/sessions.csv', 'false'], ['logs/session-costs.json', 'false'], ['shots/session.PNG', 'false'], + ['src/auth/session.ts', 'true'], ['lib/session-store.ts', 'true'], ['app/controllers/sessions_controller.rb', 'true'], ['config/sessions.json', 'true'], + ['.github/session.json', 'true'], ['app/.session.json', 'true'], ['session.config.json', 'true'], ['data/oauth-tokens.json', 'true'], + ] as const) { + const fx = fixture(); + fx.commit({ [file]: '{"a": 1}\n', 'src/run.ts': 'export const run = () => 2;\n' }); + expect([file, fx.scope(['main']).vars.SCOPE_AUTH]).toEqual([file, auth]); + } + }, 120_000); + + // Value: protects=the planner adds no auth reader and no gate class for a session-named data file, and a credential in it is still a secret; + // fails_when=targetedClasses returns auth for recovered-sessions.jsonl next to a code change, classifyDiff raises it to gate through SCOPE_AUTH, + // or the secret class misses an env key reference in that file; seam=gstack-diff-scope + collectCommit + targetedClasses + depthOfCommitInput + test('the planner: no auth class and no gate for session rows; real session code is auth; a key reference is a secret', () => { + const fx = fixture(); + const rows = Array.from({ length: 20 }, (_, i) => `{"run": ${i}, "tokens": ${1000 + i}}`).join('\n') + '\n'; + const run = (files: Record) => { + const head = fx.commit(files); + const c = collectCommit(fx.work, fx.base, head); + const scope = fx.scope(['main', '--head', head]).vars; + return { targeted: targetedClasses({ ...c, scope, gateTagged: 0 }).classes, class: depthOfCommitInput({ ...c, scope, gateTagged: 0 }).class }; + }; + expect(run({ 'results/recovered-sessions.jsonl': rows, 'src/run.ts': 'export const run = () => 3;\n' })).toEqual({ targeted: [], class: 'code' }); + expect(run({ 'lib/session-store.ts': 'export const cookie = (sid: string) => `sid=${sid}; HttpOnly`;\n' })).toEqual({ targeted: ['auth'], class: 'gate' }); + expect(run({ 'results/recovered-sessions.jsonl': rows + '{"key": "${OPENAI_API_KEY}"}\n' }).targeted).toEqual(['auth', 'secret']); + }, 120_000); +}); + +describe('GST-884: gstack-diff-scope and the receipt depth read origin/ by its full ref name', () => { + function staleFixture() { + const fx = fixture(); + fx.sh(fx.root, 'git clone -q origin.git other && cd other && mkdir -p db/migrate && echo "create table t;" > db/migrate/001.sql && git add -A && git commit -qm upstream && git push -q origin main'); + fx.sh(fx.work, 'git fetch -q origin && git checkout -q -B feat origin/main && echo "a{}" > styles.css && git add -A && git commit -qm feat'); + return fx; + } + + // Value: protects=a local main behind origin/main never adds upstream files to the scope; fails_when=gstack-diff-scope diffs against the + // stale local main and reports SCOPE_MIGRATIONS for a migration the branch only inherited; seam=bin/gstack-diff-scope + test('a stale local main scopes the same as a fresh one', () => { + const fx = staleFixture(); + expect(fx.sh(fx.work, 'git merge-base main HEAD')).not.toBe(fx.sh(fx.work, 'git merge-base origin/main HEAD')); + const stale = fx.scope(['main']); + fx.sh(fx.work, 'git branch -f main origin/main'); + const fresh = fx.scope(['main']); + expect(fresh.vars).toMatchObject({ SCOPE_FRONTEND: 'true', SCOPE_MIGRATIONS: 'false' }); + expect(stale.out).toBe(fresh.out); + }, 60_000); + + // Value: protects=a tag named main or origin/main cannot stand in for the branch and hide the change; fails_when=gstack-diff-scope resolves + // the base by its short name while a tag of that name exists at HEAD (an empty scope); seam=bin/gstack-diff-scope + for (const [tag, arg] of [['main', 'main'], ['origin/main', 'main'], ['origin/main', 'origin/main']] as const) { + test(`a tag named ${tag} at HEAD does not shadow the base ${arg}`, () => { + const fx = staleFixture(); + fx.sh(fx.work, 'git branch -f main origin/main'); + const before = fx.scope([arg]); + fx.sh(fx.work, `git tag '${tag}' HEAD`); + const after = fx.scope([arg]); + expect(before.vars.SCOPE_FRONTEND).toBe('true'); + expect(after.out).toBe(before.out); + const head = fx.sh(fx.work, 'git rev-parse HEAD'); + expect(fx.scope([arg, '--head', head]).out).toBe(before.out); + }, 60_000); + } + + // Value: protects=the receipt's depth and the queue's recomputation diff against refs/remotes/origin/, never a tag named origin/main; + // fails_when=receiptDepthAtHead or receiptDepthVerdict rev-parse the short name origin/; seam=lib/ship-receipt-depth.ts + test('a tag named origin/main does not move the receipt depth base', () => { + const fx = staleFixture(); + const origin = fx.sh(fx.work, 'git rev-parse refs/remotes/origin/main'); + const head = fx.sh(fx.work, 'git rev-parse HEAD'); + fx.sh(fx.work, "git tag 'origin/main' HEAD && git tag main HEAD"); + const stub = path.join(fx.root, 'stub-bin'); + fs.mkdirSync(stub); + fs.writeFileSync(path.join(stub, 'gstack-diff-scope'), '#!/bin/sh\nexit 0\n', { mode: 0o755 }); + fs.writeFileSync(path.join(stub, 'gstack-gate-diff'), '#!/bin/sh\nexit 1\n', { mode: 0o755 }); + expect(receiptDepthAtHead(fx.work, 'main', stub, head).base).toBe(origin); + const verdict = receiptDepthVerdict(fx.work, 'origin/main', { gstack_version: '1.0.0', review_depth: { depth: 'full', gate_scan: { tagged: 0, head } } } as any, head); + expect(verdict.line).toContain(`inputs=base@${origin.slice(0, 12)}`); + expect(verdict.line).toContain('class=code'); + }, 60_000); +});