mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-09 18:28:15 +01:00
v1.94.1.0 stamp: merge main 4ec1722ecb into #3893 (land)
This commit is contained in:
commit
adcf98741c
66 files changed
+2392
-193
No files matched your search
@@ -0,0 +1,111 @@
|
||||
/**
|
||||
* lib/ci-carry.ts (GSTA-258 piece 1; design fixes A2, A3): CI's verified run at HEAD's exact, clean tree stands in
|
||||
* for running a lane again. The dependencies are fakes, so nothing here reaches GitHub.
|
||||
*/
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import { findCarry, shipLanesErrors, waitForRun } from '../lib/ci-carry';
|
||||
import { validateConfig } from '../lib/pregate/config';
|
||||
import { receiptSummaryLine, validateReceipt } from '../lib/ship-receipt';
|
||||
import type { ApiJob, ApiRun, RunDeps, RunProfile, RunSource, TreeRecord } from '../lib/run-evidence';
|
||||
|
||||
const REPO = 'garrytan/gstack';
|
||||
const HEAD = 'a'.repeat(40);
|
||||
const TREE = 'b'.repeat(40);
|
||||
const NOW = new Date('2026-11-02T12:00:00Z');
|
||||
const PROFILE: RunProfile = { workflow: 'free-tests.yml', events: ['pull_request', 'workflow_dispatch'], jobs: ['free-tests'], artifact: 'tested-tree', os: 'Linux' };
|
||||
|
||||
const run = (id: number, over: Partial<ApiRun> = {}): ApiRun => ({
|
||||
id, run_attempt: 1, status: 'completed', conclusion: 'success', event: 'workflow_dispatch', head_sha: HEAD, path: '.github/workflows/free-tests.yml',
|
||||
created_at: `2026-11-02T1${id % 10}:00:00Z`.replace('T1', 'T0'), updated_at: '2026-11-02T11:30:00Z', repository: { full_name: REPO }, head_repository: { full_name: REPO }, ...over,
|
||||
});
|
||||
const record = (id: number, over: Partial<TreeRecord> = {}): TreeRecord => ({ run_id: id, run_attempt: 1, event: 'workflow_dispatch', sha: HEAD, tree: TREE, files: ['test/a.test.ts', 'test/b.test.ts'], tests: 40, os: 'Linux', ...over });
|
||||
function source(runs: ApiRun[], records: Record<number, TreeRecord> = {}, jobs: ApiJob[] = [{ name: 'free-tests', conclusion: 'success', completed_at: '2026-11-02T11:30:00Z' }]): RunSource {
|
||||
const deps: RunDeps = {
|
||||
run: id => { const r = runs.find(x => x.id === id); if (!r) throw new Error('HTTP 404'); return r; },
|
||||
runsForHead: (_wf, sha) => runs.filter(r => r.head_sha === sha),
|
||||
jobs: () => jobs,
|
||||
artifact: id => { if (!records[id]) throw new Error('no artifact'); return records[id]!; },
|
||||
commitTree: sha => (sha === HEAD ? TREE : null),
|
||||
};
|
||||
return { repo: REPO, deps, profiles: { 'ci-reuse/free-tests': PROFILE }, now: NOW };
|
||||
}
|
||||
const clean = () => TREE;
|
||||
const carry = (src: RunSource, over: Partial<Parameters<typeof findCarry>[1]> = {}) => findCarry(src, { profiles: ['ci-reuse/free-tests'], head: HEAD, tree: TREE, wtree: clean, ...over });
|
||||
const why = (v: ReturnType<typeof findCarry>) => (v.ok ? 'ok' : v.reason);
|
||||
|
||||
describe('findCarry', () => {
|
||||
// Value: protects=a green verified run at this exact tree carries, with a receipt naming it; fails_when=a verified run is ignored or its count is lost;
|
||||
// why_new=GSTA-258 piece 1; seam=lib/run-evidence.ts verifyRun; no_claim=GitHub's API shape (ghRunDeps)
|
||||
test('the newest verified run at HEAD carries, and the receipt names its run and tree', () => {
|
||||
const v = findCarry(source([run(4)], { 4: record(4) }), { profiles: ['ci-reuse/free-tests'], head: HEAD, tree: TREE, wtree: clean, inventory: () => ['test/a.test.ts', 'test/b.test.ts'] });
|
||||
expect(v.ok).toBe(true);
|
||||
if (!v.ok) return;
|
||||
expect(v.tests).toBe(40);
|
||||
expect(v.runs.map(e => e.run_id)).toEqual([4]);
|
||||
expect(v.receipt).toMatch(/^ci-carry tree=bbbbbbbbbbbb tests=40 runs=4\/1 profiles=ci-reuse\/free-tests heads=aaaaaaaaaaaa verified=/);
|
||||
});
|
||||
|
||||
// Value: protects=A2, a carry binds to the working-tree fingerprint before and after; fails_when=a dirty tree or a tree edited during the lookup carries
|
||||
test('a dirty working tree, or one that changes while runs are read, never carries', () => {
|
||||
expect(why(carry(source([run(4)], { 4: record(4) }), { wtree: () => 'c'.repeat(40) }))).toBe('dirty_tree');
|
||||
expect(why(carry(source([run(4)], { 4: record(4) }), { wtree: () => null }))).toBe('dirty_tree');
|
||||
let reads = 0;
|
||||
expect(why(carry(source([run(4)], { 4: record(4) }), { wtree: () => (reads++ ? 'c'.repeat(40) : TREE) }))).toBe('tree_changed');
|
||||
});
|
||||
|
||||
// Value: protects=A3, a lane carries only what CI provably ran; fails_when=a tree.json missing an inventory file still carries
|
||||
test('the inventory must be covered by the runs\' tree.json files', () => {
|
||||
const src = source([run(4)], { 4: record(4) });
|
||||
const v = carry(src, { inventory: () => ['test/a.test.ts', 'test/c.test.ts'] });
|
||||
expect(why(v)).toBe('inventory_uncovered');
|
||||
expect(v.ok ? '' : v.detail).toContain('test/c.test.ts');
|
||||
expect(why(carry(src, { inventory: () => ({ error: 'exited 1' }) }))).toBe('inventory_failed');
|
||||
expect(why(carry(src, { inventory: () => [] }))).toBe('inventory_empty');
|
||||
});
|
||||
|
||||
// Value: protects=fork runs, newer red or running runs, stale runs and empty runs never carry; fails_when=any of them is accepted
|
||||
test('no run, a fork run, a newer red or running run, a stale run and a run with no tests are refused', () => {
|
||||
expect(why(carry(source([])))).toBe('no_run');
|
||||
expect(why(carry(source([run(4, { head_repository: { full_name: 'evil/gstack' } })], { 4: record(4) })))).toBe('no_verified_run');
|
||||
const red = carry(source([run(4), run(6, { conclusion: 'failure' })], { 4: record(4), 6: record(6) }));
|
||||
expect(why(red)).toBe('no_verified_run');
|
||||
expect(red.ok ? '' : red.detail).toContain('run 6: run_not_success; run 4: newer_run_vetoes');
|
||||
expect(why(carry(source([run(4), run(6, { status: 'in_progress', conclusion: null })], { 4: record(4) })))).toBe('no_verified_run');
|
||||
expect(why(carry(source([run(4)], { 4: record(4) }, [{ name: 'free-tests', conclusion: 'success', completed_at: '2026-10-30T00:00:00Z' }])))).toBe('no_verified_run');
|
||||
expect(why(carry(source([run(4)], { 4: record(4, { files: [] }) })))).toBe('no_tests');
|
||||
expect(why(carry(source([run(4)], { 4: record(4) }), { profiles: ['nope'] }))).toBe('no_profile');
|
||||
});
|
||||
});
|
||||
|
||||
describe('waitForRun', () => {
|
||||
const clockAt = () => { let t = 0; return { clock: () => t, sleep: async (ms: number) => { t += ms; } }; };
|
||||
// Value: protects=the bounded wait reports green, red with failing jobs, still running and none; fails_when=it waits forever or calls a red run green
|
||||
test('green, red with its failing jobs, still running at the deadline, and none', async () => {
|
||||
const o = (runs: ApiRun[], jobs?: ApiJob[]) => waitForRun(source(runs, {}, jobs).deps, { workflow: 'free-tests.yml', events: PROFILE.events, head: HEAD, timeoutMs: 60_000, intervalMs: 20_000, ...clockAt() });
|
||||
expect((await o([run(4)])).state).toBe('green');
|
||||
const red = await o([run(4, { conclusion: 'failure' })], [{ name: 'free-tests', conclusion: 'failure' }, { name: 'typecheck', conclusion: 'success' }]);
|
||||
expect(red).toMatchObject({ state: 'red', failing: ['free-tests'] });
|
||||
expect((await o([run(4, { status: 'in_progress', conclusion: null })])).state).toBe('running');
|
||||
expect((await o([])).state).toBe('none');
|
||||
});
|
||||
});
|
||||
|
||||
describe('ship_lanes_from config and the ci_carried receipt field', () => {
|
||||
// Value: protects=A3, the mapping names real profiles and one inventory command; fails_when=a typo or a label-shaped entry validates
|
||||
test('ship_lanes_from validates its profiles and inventory', () => {
|
||||
const profiles = { 'ci-reuse/free-tests': PROFILE };
|
||||
expect(validateConfig({ run_profiles: profiles, ship_lanes_from: { 'bun run test': { profiles: ['ci-reuse/free-tests'], inventory: 'bun run list' } } })).toEqual([]);
|
||||
expect(shipLanesErrors({ 'bun run test': { profiles: ['nope'], inventory: 'x' } }, ['ci-reuse/free-tests']).map(e => e.message)).toEqual(['"nope" is not a run_profiles name']);
|
||||
expect(shipLanesErrors({ 'bun run test': { profiles: ['ci-reuse/free-tests'], inventory: 'a\nb' } }, ['ci-reuse/free-tests'])[0]!.path).toBe('ship_lanes_from["bun run test"].inventory');
|
||||
expect(shipLanesErrors({ 'bun run test': { profiles: ['ci-reuse/free-tests'] , inventory: 'x', label: 'free' } }, ['ci-reuse/free-tests'])[0]!.message).toContain('unknown key');
|
||||
expect(shipLanesErrors(['bun run test'], [])[0]!.path).toBe('ship_lanes_from');
|
||||
});
|
||||
|
||||
// Value: protects=the receipt and its SHIP_RECEIPT line name the carried runs; fails_when=ci_carried is dropped or malformed entries validate
|
||||
test('the receipt records ci_carried and its summary line names each run', () => {
|
||||
const receipt = { schema_version: 1 as const, head: HEAD, base: 'main', ci_carried: [{ lane: 'free', run: 41, attempt: 2, tree: TREE }] };
|
||||
expect(validateReceipt(receipt)).toEqual([]);
|
||||
expect(receiptSummaryLine(receipt)).toContain('ci_carried=free:41/2');
|
||||
expect(validateReceipt({ ...receipt, ci_carried: [{ lane: 'free', run: '41' }] }).length).toBeGreaterThan(0);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,141 @@
|
||||
/**
|
||||
* gstack-evidence reuse and CI-sourced rows (GSTA-258 pieces 1-2, design fixes A4 and A5).
|
||||
*
|
||||
* run --reuse: a lane that passed at this exact working tree, command and execution identity never runs again;
|
||||
* the newest matching row decides (a newer failure vetoes), the reused row keeps its original timestamp and the
|
||||
* max age applies to it. record-ci: a lane CI carried is an explicit CI-sourced row for a run it verified itself, never a fictitious local run.
|
||||
*/
|
||||
import { afterEach, beforeEach, describe, expect, setDefaultTimeout, test } from 'bun:test';
|
||||
import { createHash } from 'node:crypto';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { findFilesBySuffix, gitIn } from './helpers/scratch-repo';
|
||||
import { runBin } from './helpers/run-bin';
|
||||
|
||||
setDefaultTimeout(60_000);
|
||||
const EVIDENCE = path.join(import.meta.dir, '..', 'bin', 'gstack-evidence');
|
||||
let home: string;
|
||||
let repo: string;
|
||||
let counter: string;
|
||||
|
||||
const run = (args: string[]) => runBin(EVIDENCE, args, { cwd: repo, env: { GSTACK_HOME: home } });
|
||||
const ledger = () => findFilesBySuffix(path.join(home, 'projects'), '-evidence.jsonl')[0]!;
|
||||
const rows = (): any[] => fs.readFileSync(ledger(), 'utf8').trim().split('\n').map(l => JSON.parse(l));
|
||||
const rewrite = (edit: (r: any[]) => any[]) => fs.writeFileSync(ledger(), edit(rows()).map(r => JSON.stringify(r)).join('\n') + '\n');
|
||||
const executions = () => (fs.existsSync(counter) ? fs.readFileSync(counter, 'utf8').length : 0);
|
||||
const lane = (pass: number, fail = 0) => `printf x >> '${counter}'; printf ' ${pass} pass\\n ${fail} fail\\nRan ${pass + fail} tests across 1 file. [1.00ms]\\n' >&2; exit ${fail ? 1 : 0}`;
|
||||
const wtree = () => runBin(path.join(import.meta.dir, '..', 'bin', 'gstack-wtree'), [], { cwd: repo }).stdout.trim();
|
||||
|
||||
beforeEach(() => {
|
||||
home = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-reuse-home-'));
|
||||
repo = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-reuse-repo-'));
|
||||
counter = path.join(home, 'executions');
|
||||
gitIn(repo, 'init -q -b main');
|
||||
fs.writeFileSync(path.join(repo, 'src.txt'), 'v1\n');
|
||||
gitIn(repo, 'add src.txt');
|
||||
gitIn(repo, 'commit -q -m init');
|
||||
});
|
||||
afterEach(() => {
|
||||
fs.rmSync(home, { recursive: true, force: true });
|
||||
fs.rmSync(repo, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
describe('gstack-evidence run --reuse (A5)', () => {
|
||||
// Value: protects=a lane that passed at this tree never reruns, and the reuse is visible; fails_when=--reuse runs the suite again, or reports a pass it never recorded;
|
||||
// why_new=GSTA-258 piece 2; seam=the ledger and the wtree fingerprint; no_claim=cross-machine reuse (CI carries that)
|
||||
test('a pass at the same tree, command and identity is reused with its original timestamp and count', () => {
|
||||
expect(run(['run', '--label', 'unit', '--', lane(4)]).status).toBe(0);
|
||||
const first = rows().at(-1);
|
||||
const again = run(['run', '--reuse', '--label', 'unit', '--', lane(4)]);
|
||||
expect(again.status).toBe(0);
|
||||
expect(executions()).toBe(1);
|
||||
expect(again.stderr).toContain(`gstack-evidence: reused label=unit tree=${wtree().slice(0, 12)} from=${first.ts}`);
|
||||
expect(again.stderr).toMatch(/gstack-evidence: recorded label=unit exit=0 tests_ran=4 log=\S+/);
|
||||
const copy = rows().at(-1);
|
||||
expect(copy.ts).toBe(first.ts);
|
||||
expect(copy.reused_from).toBe(first.ts);
|
||||
expect(copy.identity).toBe(first.identity);
|
||||
expect(typeof first.identity).toBe('string');
|
||||
expect(run(['check', '--label', 'unit']).stdout).toContain('EVIDENCE: FRESH label=unit exit=0 tests_ran=4');
|
||||
});
|
||||
|
||||
// Value: protects=only the newest matching row decides; fails_when=an older pass is reused under a newer failure on the same tree
|
||||
test('a newer failure on the same tree vetoes an older pass; without --reuse nothing is reused', () => {
|
||||
expect(run(['run', '--label', 'unit', '--', lane(4)]).status).toBe(0);
|
||||
expect(run(['run', '--label', 'unit', '--', lane(3, 1)]).status).toBe(1);
|
||||
expect(run(['run', '--reuse', '--label', 'unit', '--', lane(3, 1)]).status).toBe(1);
|
||||
expect(executions()).toBe(3);
|
||||
expect(run(['run', '--label', 'unit', '--', lane(4)]).status).toBe(0);
|
||||
expect(run(['run', '--label', 'unit', '--', lane(4)]).status).toBe(0);
|
||||
expect(executions()).toBe(5);
|
||||
});
|
||||
|
||||
// Value: protects=reuse binds to the exact command, tree, count and identity; fails_when=a changed command, an edited file, a zero-run or unknown row, or another runtime reuses
|
||||
test('another command, an edited tree, a zero-run or unknown count, or another identity runs again', () => {
|
||||
expect(run(['run', '--label', 'unit', '--', lane(4)]).status).toBe(0);
|
||||
expect(run(['run', '--reuse', '--label', 'unit', '--', lane(5)]).status).toBe(0);
|
||||
expect(executions()).toBe(2);
|
||||
fs.writeFileSync(path.join(repo, 'src.txt'), 'v2\n');
|
||||
expect(run(['run', '--reuse', '--label', 'unit', '--', lane(5)]).status).toBe(0);
|
||||
expect(executions()).toBe(3);
|
||||
rewrite(r => r.map(x => ({ ...x, tests_ran: 0 })));
|
||||
expect(run(['run', '--reuse', '--label', 'unit', '--', lane(5)]).status).toBe(0);
|
||||
expect(executions()).toBe(4);
|
||||
rewrite(r => r.map(x => ({ ...x, tests_ran: 'unknown' })));
|
||||
expect(run(['run', '--reuse', '--label', 'unit', '--', lane(5)]).status).toBe(0);
|
||||
expect(executions()).toBe(5);
|
||||
rewrite(r => r.map(x => ({ ...x, identity: 'f'.repeat(64) })));
|
||||
expect(run(['run', '--reuse', '--label', 'unit', '--', lane(5)]).status).toBe(0);
|
||||
expect(executions()).toBe(6);
|
||||
});
|
||||
|
||||
// Value: protects=the max age applies to the original pass, never to a refreshed copy; fails_when=reuse makes an old pass look new
|
||||
test('a pass older than --max-age is not reused, and a reused copy keeps the age check honest', () => {
|
||||
expect(run(['run', '--label', 'unit', '--', lane(4)]).status).toBe(0);
|
||||
rewrite(r => r.map(x => ({ ...x, ts: new Date(Date.now() - 30 * 3600_000).toISOString() })));
|
||||
expect(run(['run', '--reuse', '--label', 'unit', '--', lane(4)]).status).toBe(0);
|
||||
expect(executions()).toBe(2);
|
||||
rewrite(r => r.map(x => ({ ...x, ts: new Date(Date.now() - 2 * 3600_000).toISOString() })));
|
||||
expect(run(['run', '--reuse', '--max-age', '1', '--label', 'unit', '--', lane(4)]).status).toBe(0);
|
||||
expect(executions()).toBe(3);
|
||||
rewrite(r => r.map(x => ({ ...x, ts: new Date(Date.now() - 3 * 3600_000).toISOString() })));
|
||||
expect(run(['run', '--reuse', '--label', 'unit', '--', lane(4)]).status).toBe(0);
|
||||
expect(executions()).toBe(3);
|
||||
expect(run(['check', '--label', 'unit', '--max-age', '2']).stdout).toContain('EVIDENCE: STALE label=unit');
|
||||
});
|
||||
});
|
||||
|
||||
describe('gstack-evidence record-ci and carried (A4)', () => {
|
||||
const ciRow = () => {
|
||||
const tree = wtree();
|
||||
return { ts: new Date(Date.now() - 600_000).toISOString(), label: 'free', command: 'bun run test', cmd_sha256: createHash('sha256').update('bun run test').digest('hex'), exit: 0, duration_s: 0, commit: gitIn(repo, 'rev-parse HEAD').trim(), tree, dirty: false, wtree: tree, tests_ran: 900, source: 'ci', runs: ['41/2'], profiles: ['ci-reuse/free-tests'] };
|
||||
};
|
||||
|
||||
// Value: protects=a carried lane is an explicit CI row the FRESH check accepts by source, and the receipt lists it only at its own tree;
|
||||
// fails_when=a check cannot tell a CI row from a local run, or a stale carry from an earlier head reaches ci_carried
|
||||
test('a CI-sourced row is FRESH for its exact command, and carried lists it only at its tree', () => {
|
||||
expect(run(['run', '--label', 'unit', '--', lane(2)]).status).toBe(0);
|
||||
fs.appendFileSync(ledger(), JSON.stringify(ciRow()) + '\n');
|
||||
const chk = run(['check', '--label', 'free', '--expect-cmd', 'bun run test', '--max-age', '24']);
|
||||
expect(chk.status).toBe(0);
|
||||
expect(chk.stdout).toContain('EVIDENCE: FRESH label=free exit=0 tests_ran=900');
|
||||
expect(chk.stdout).toContain('source=ci runs=41/2');
|
||||
expect(JSON.parse(run(['carried', '--json']).stdout)).toEqual([{ lane: 'free', run: 41, attempt: 2, tree: wtree() }]);
|
||||
fs.writeFileSync(path.join(repo, 'src.txt'), 'v2\n');
|
||||
gitIn(repo, 'commit -qam two');
|
||||
expect(JSON.parse(run(['carried', '--json']).stdout)).toEqual([]);
|
||||
});
|
||||
|
||||
// Value: protects=record-ci finds and verifies the CI run itself, so no caller supplies run ids, a test count or a tree (/ship pass 1 red-team#1);
|
||||
// fails_when=the forged call from that finding writes a FRESH carried row; seam=no origin, so the base's ship_lanes_from is unreadable
|
||||
test('record-ci takes no run facts from its caller and records nothing without a verified run', () => {
|
||||
const forged = run(['record-ci', '--label', 'free', '--runs', '1/1', '--profiles', 'anything', '--tree', wtree(), '--tests', '999', '--finished', new Date().toISOString(), '--', 'bun run test']);
|
||||
expect(forged.status).toBe(2);
|
||||
expect(forged.stderr).toContain('usage: gstack-evidence record-ci --label <L> --base <base> -- <command>');
|
||||
const unverified = run(['record-ci', '--label', 'free', '--base', 'main', '--', 'bun run test']);
|
||||
expect(unverified.status).toBe(1);
|
||||
expect(unverified.stderr).toMatch(/^CARRY_NONE: config_unavailable: /m);
|
||||
expect(findFilesBySuffix(path.join(home, 'projects'), '-evidence.jsonl')).toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,6 @@
|
||||
REVIEWED_TREE: 4d6a982968de8e11d2665976fc91587730a4767d /home/user/.capy/work/gst986-wt
|
||||
{"severity":"INFORMATIONAL","path":"docs/troubleshooting.md","line":3307,"summary":"The RECEIPT_INVALID entry still describes the old AND rule: it says write refuses when the read commit's product patch is not the receipt's AND a later change carries a targeted class. The fixed code at lib/ship-receipt-depth.ts:153 refuses on either one. So a user who hits the refusal after a docs-only or TODOS.md commit (the Step 14 and 14.5 commits) finds no matching cause in this entry. docs/review-policy.md and the changelog fragment were updated; this entry was not."}
|
||||
{"severity":"INFORMATIONAL","path":"lib/ship-receipt-depth.ts","line":153,"summary":"The fix list calls this main's rule, but it isn't. On origin/main, parseReviewTargeted(value, productPatchId) takes no head and binds any record. So under this change, every money, gate, auth or secret ship whose Step 14 or 14.5 commits after the Step 11 read now stops at Step 19 with RECEIPT_INVALID. That is the same outcome pass 1's Red Team flagged. Step 19's prose (ship/sections/pr-body.md:247, 'fix that input file and rerun') doesn't say to rerun the targeted reader on HEAD; only the tool's own error message does. The guard fails closed, and the code deliberately defers the docs-only case to GST-1031, so this is not data loss. It is a known extra stop in unattended /ship runs."}
|
||||
{"severity":"INFORMATIONAL","path":"lib/ship-receipt-depth.ts","line":153,"summary":"When the product patches are equal and only classesSince refuses (a test edit after the read that skips a case), the message prints the same patch id twice: 'whose product patch X is not this receipt's X'. This was already true before the fix and the fix kept it. The message misstates the cause; the decision itself is correct."}
|
||||
|
||||
I found nothing at the blocking floor. The OR guard fails closed in the right places: it refuses when the product patch differs, when classesSince returns null (unreadable), and when a class shows up after the read. Its only production caller, bin/gstack-ship-receipt:227, passes the same arguments as before. The test change does replace the old docs-binds assertion with a refusal, but that matches the intended new rule; it doesn't weaken coverage. The new no-money-token regression test covers the CRITICAL fix. I reran the three focused files at this tree: 34 pass, 0 fail, 403 expect() calls.
|
||||
+34
@@ -0,0 +1,34 @@
|
||||
I reviewed the range base..HEAD, one commit that changes four files. I read every non-test file in full.
|
||||
|
||||
## Findings
|
||||
|
||||
**1. The family check refuses two models in one call even when no task is judged (check.ts:165-169). FIXABLE.**
|
||||
The family check runs whether or not the run uses a judge. Elsewhere the judge is used only when `judged` is true.
|
||||
|
||||
Before this commit the call passed. Now every judge stops with E_POLICY:
|
||||
- the default judge clashes with the second model;
|
||||
- the other judges clash with the first model.
|
||||
|
||||
So the two models can no longer run in one call, even though the judge would never be called. It fails closed with a clear message, and two calls work around it.
|
||||
Fix: `else if (judged && sameFamily.length)`, plus a test with an unjudged set.
|
||||
Severity: Medium. Blocks: no.
|
||||
|
||||
**2. max_price now uses the long-prompt rates for every call, short prompts included (prices.ts:75, set at proxy.ts:513). FIXABLE, with part to INVESTIGATE.**
|
||||
`routing` sets max_price from the price row alone, whatever the prompt length.
|
||||
|
||||
Today the only endpoints are standard, fast and ultrafast.
|
||||
|
||||
Two more problems:
|
||||
- The commit assumes the router checks max_price against long-tier rates.
|
||||
- The new doc comment gives the wrong reason.
|
||||
|
||||
Fix: build max_price per request.
|
||||
Severity: Low. Blocks: no.
|
||||
|
||||
**3. The docs' third-family judge barely exists (docs/eval.md:125, policy.ts:125-129). INVESTIGATE.**
|
||||
Only weak judges have price rows, so in practice the remedy is two calls.
|
||||
|
||||
Also, call.ts still picks the default judge by provider, not by family.
|
||||
Severity: Low. Blocks: no.
|
||||
|
||||
Recommendation: Fix the unconditional family check at check.ts:169, because gating it on `judged` is a one-line change.
|
||||
@@ -42,6 +42,7 @@ const KEPT: Record<string, string> = {
|
||||
'review-*': '/ship review loop records',
|
||||
'fix-check-*': '/ship fix-check inputs',
|
||||
'tasks-*': 'gstack-tasks rows files (bin/gstack-tasks)',
|
||||
'qa-reports': 'caller QA report runs from gstack-qa-evidence report-dir (lib/qa-evidence.ts)',
|
||||
'autoplan': 'autoplan phase artifacts (autoplan/SKILL.md.tmpl)',
|
||||
'eval-bisect': '/investigate eval bisect cache',
|
||||
'price-gate': 'gstack-eval-plan price receipts',
|
||||
|
||||
@@ -225,12 +225,29 @@ describe('carry follow-ups: git floor, named refusal, branch-touched walk, digit
|
||||
const mine = "export const MINE_KEY_VERSION = '1.2.3.0';";
|
||||
const before = `${mine}\nexport const BASE_KEY_VERSION = '1.2.3.0';\n`;
|
||||
const moved = (text: string) => ({ before, after: text, oldVersion: '1.2.4.0', newVersion: '1.2.5.0', templates });
|
||||
const side = { fork: '1.2.3.0', added: new Set([mine]) };
|
||||
const side = { fork: '1.2.3.0', added: new Set([mine]), at: new Set([1]) };
|
||||
expect(stampOnlyText({ ...moved(before.replace(mine, mine.replace('1.2.3.0', '1.2.5.0'))), side })).toBe(true);
|
||||
expect(stampOnlyText(moved(before.replace(mine, mine.replace('1.2.3.0', '1.2.5.0'))))).toBe(false);
|
||||
expect(stampOnlyText({ ...moved(before.replace("BASE_KEY_VERSION = '1.2.3.0'", "BASE_KEY_VERSION = '1.2.5.0'")), side })).toBe(false);
|
||||
});
|
||||
|
||||
// Value: protects=the carry reads the branch's own line by position and, with a kept VERSION claim, accepts only the
|
||||
// branch's own row moved from the fork version (GST-979, GST-980); fails_when=stampOnlyText refuses every change at
|
||||
// equal versions or takes a duplicated base line as the branch's; seam=lib/ship-receipt.ts stampOnlyText
|
||||
test('stampOnlyText reads the branch line by position, and with a kept VERSION moves only the branch\'s own row (GST-979, GST-980)', () => {
|
||||
const templates = ["_KEY_VERSION = '{version}'"];
|
||||
const key = (v: string) => `export const MINE_KEY_VERSION = '${v}';`;
|
||||
const before = `${key('1.2.3.0')}\nexport const BASE_KEY_VERSION = '1.2.3.0';\n${key('1.2.3.0')}\n`;
|
||||
const side = { fork: '1.2.3.0', added: new Set([key('1.2.3.0')]), at: new Set([3]) };
|
||||
const lines = (a: string, c: string) => `${a}\nexport const BASE_KEY_VERSION = '1.2.3.0';\n${c}\n`;
|
||||
const change = (after: string, oldVersion: string, newVersion: string) => stampOnlyText({ before, after, oldVersion, newVersion, templates, side });
|
||||
expect(change(lines(key('1.2.3.0'), key('1.2.5.0')), '1.2.4.0', '1.2.5.0')).toBe(true);
|
||||
expect(change(lines(key('1.2.5.0'), key('1.2.3.0')), '1.2.4.0', '1.2.5.0')).toBe(false);
|
||||
expect(change(lines(key('1.2.3.0'), key('1.2.4.0')), '1.2.4.0', '1.2.4.0')).toBe(true);
|
||||
expect(change(lines(key('1.2.4.0'), key('1.2.3.0')), '1.2.4.0', '1.2.4.0')).toBe(false);
|
||||
expect(stampOnlyText({ before, after: lines(key('1.2.3.0'), key('1.2.4.0')), oldVersion: '1.2.4.0', newVersion: '1.2.4.0', templates })).toBe(false);
|
||||
});
|
||||
|
||||
test('planCarry names at most CARRY_NAMED_PATHS paths', () => {
|
||||
const many = Array.from({ length: CARRY_NAMED_PATHS + 3 }, (_, i) => `src/f${i}.ts`);
|
||||
const plan = planCarry({ from: 'a'.repeat(40), head: 'b'.repeat(40), changed: many.map(p => ({ path: p, class: 'product' as const })), productPatchId: { from: 'x', head: 'y' }, testPatchId: { from: '', head: '' }, changedProduct: many, mergeCommits: 0, rangeDiff: '' });
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
/**
|
||||
* A fake gh for the CI-carry tests (GSTA-258): serves the Actions run, jobs, artifact and download reads
|
||||
* lib/run-evidence.ts ghRunDeps makes from a world file, and answers every other gh call with nothing.
|
||||
*/
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
|
||||
export interface FakeWorld {
|
||||
runs: Array<Record<string, unknown> & { id: number; run_attempt: number; head_sha: string; path: string }>;
|
||||
jobs: Record<string, Array<{ name: string; conclusion: string | null; completed_at?: string }>>;
|
||||
trees: Record<string, Record<string, unknown>>;
|
||||
}
|
||||
|
||||
/** The stub body for shipFixture's gh(): run this file with the world beside it. */
|
||||
export const fakeGhScript = (worldFile: string) => `exec bun --no-env-file --config=/dev/null '${import.meta.path}' '${worldFile}' "$@"`;
|
||||
|
||||
if (import.meta.main) {
|
||||
const [worldFile, ...a] = process.argv.slice(2);
|
||||
const world: FakeWorld = fs.existsSync(worldFile!) ? JSON.parse(fs.readFileSync(worldFile!, 'utf8')) : { runs: [], jobs: {}, trees: {} };
|
||||
const lines = (list: unknown[]) => process.stdout.write(list.map(x => JSON.stringify(x)).join('\n') + (list.length ? '\n' : ''));
|
||||
if (a[0] === 'api') {
|
||||
const jq = a.indexOf('--jq');
|
||||
const p = a.filter((x, i) => i > 0 && x !== '--paginate' && i !== jq && i !== jq + 1)[0] ?? '';
|
||||
const [route, query = ''] = p.split('?');
|
||||
const q = new URLSearchParams(query);
|
||||
let m: RegExpExecArray | null;
|
||||
if ((m = /\/actions\/runs\/(\d+)$/.exec(route!))) {
|
||||
const run = world.runs.find(r => r.id === Number(m![1]));
|
||||
if (!run) { process.stderr.write('HTTP 404'); process.exit(1); }
|
||||
process.stdout.write(JSON.stringify(run));
|
||||
} else if ((m = /\/actions\/workflows\/([^/]+)\/runs$/.exec(route!))) {
|
||||
lines(world.runs.filter(r => r.head_sha === q.get('head_sha') && r.path === `.github/workflows/${m![1]}`));
|
||||
} else if ((m = /\/actions\/runs\/(\d+)\/attempts\/\d+\/jobs$/.exec(route!))) {
|
||||
lines(world.jobs[m[1]!] ?? []);
|
||||
} else if ((m = /\/actions\/runs\/(\d+)\/artifacts$/.exec(route!))) {
|
||||
lines(world.trees[m[1]!] ? [{ name: q.get('name'), size_in_bytes: 200, expired: false }] : []);
|
||||
} else { process.stderr.write(`fake gh: no route for ${p}`); process.exit(1); }
|
||||
} else if (a[0] === 'run' && a[1] === 'download') {
|
||||
const tree = world.trees[a[2]!];
|
||||
if (!tree) process.exit(1);
|
||||
const dir = a[a.indexOf('--dir') + 1]!;
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
fs.writeFileSync(path.join(dir, 'tree.json'), JSON.stringify(tree));
|
||||
}
|
||||
}
|
||||
@@ -8,7 +8,7 @@ import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { classifyOutsideReview, validateOutsideReview, type OutsideGate } from '../lib/outside-review-result';
|
||||
import { classifyOutsideReview, usageFromEvents, validateOutsideReview, type OutsideGate } from '../lib/outside-review-result';
|
||||
import { gateOutcomeLine, type GateReason } from '../lib/gate-outcomes';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
@@ -262,6 +262,36 @@ describe('B1: a review whose sandbox could not start is unavailable, not clean',
|
||||
});
|
||||
});
|
||||
|
||||
describe('Codex token usage from --json events (#2948)', () => {
|
||||
const turn = (usage: Record<string, number>) => JSON.stringify({ type: 'turn.completed', usage });
|
||||
|
||||
test('one turn: every counter the event carries, nothing invented', () => {
|
||||
expect(usageFromEvents(turn({ input_tokens: 12016, cached_input_tokens: 0, cache_write_input_tokens: 12013, output_tokens: 5, reasoning_output_tokens: 0 })))
|
||||
.toEqual({ input_tokens: 12016, cached_input_tokens: 0, cache_write_input_tokens: 12013, output_tokens: 5, reasoning_output_tokens: 0, usd: 'unknown' });
|
||||
});
|
||||
|
||||
test('the latest turn wins (codex reports the running session total); malformed and unrelated lines are skipped', () => {
|
||||
const events = [turn({ input_tokens: 100, output_tokens: 10 }), 'not json', '{"type":"item.completed","item":{"type":"agent_message"}}', turn({ input_tokens: 150, output_tokens: 15, reasoning_output_tokens: 2 })].join('\n');
|
||||
expect(usageFromEvents(events)).toEqual({ input_tokens: 150, output_tokens: 15, reasoning_output_tokens: 2, usd: 'unknown' });
|
||||
});
|
||||
|
||||
test('a cumulative total_token_usage snapshot wins over per-turn usage, latest first', () => {
|
||||
const snap = (n: number) => JSON.stringify({ msg: { type: 'token_count', info: { total_token_usage: { input_tokens: n, output_tokens: n / 10, total_tokens: n * 1.1 } } } });
|
||||
expect(usageFromEvents([turn({ input_tokens: 7 }), snap(100), snap(300)].join('\n'))).toEqual({ input_tokens: 300, output_tokens: 30, total_tokens: 330, usd: 'unknown' });
|
||||
});
|
||||
|
||||
test('the CLI prints OUTSIDE_USAGE only when given --events, on every verdict', () => {
|
||||
const events = path.join(TMP, 'usage-events.jsonl');
|
||||
fs.writeFileSync(events, turn({ input_tokens: 1200, cached_input_tokens: 200, output_tokens: 340 }) + '\n');
|
||||
const withEvents = cli(['--events', events, 'review', file('No issues found.')]);
|
||||
expect(withEvents.stdout).toContain('OUTSIDE_USAGE: input=1200 cached_input=200 output=340\n');
|
||||
const empty = path.join(TMP, 'no-usage.jsonl');
|
||||
fs.writeFileSync(empty, '{"type":"thread.started"}\n');
|
||||
expect(cli(['--events', empty, 'review', file('')]).stdout).toContain('OUTSIDE_USAGE: unknown\n');
|
||||
expect(cli(['--verdict', 'review', file('No issues found.')]).stdout).not.toContain('OUTSIDE_USAGE');
|
||||
});
|
||||
});
|
||||
|
||||
describe('a mid-run Codex usage limit is unavailable (quota_exhausted); a mid-run 429 is unavailable (rate_limited)', () => {
|
||||
const LINE = "ERROR: You've hit your usage limit. Upgrade to Pro (https://chatgpt.com/explore/pro), visit https://chatgpt.com/codex/settings/usage to purchase more credits or try again at Oct 10th, 2026 2:55 AM.";
|
||||
const RATE_429 = 'ERROR: {"type":"error","status":429,"error":{"type":"rate_limit_exceeded","message":"Rate limit reached"}}';
|
||||
|
||||
@@ -0,0 +1,98 @@
|
||||
/**
|
||||
* CI carry and the remote gate (GSTA-258 A): after a verified CI run carries part of the tests stage on a small
|
||||
* machine, the files left over can all be ones the remote gate never runs (gstack's paid evals). The gate itself then
|
||||
* reports tests=0 (PREGATE_REMOTE_PAID_ONLY) and `clear --run` refuses paid_only, so the obligation could never clear.
|
||||
* The carried run clears it instead, through remote_gate_runs and its inventory check. The run API is a fake.
|
||||
*/
|
||||
import { afterEach, describe, expect, test } from 'bun:test';
|
||||
import type { PregateFile } from '../lib/headless-artifacts';
|
||||
import { buildContext } from '../lib/pregate/context';
|
||||
import { clearDecision, runPregate, type RunCheck } from '../lib/pregate/index';
|
||||
import type { ApiRun, RunProfile } from '../lib/run-evidence';
|
||||
import { BASE_PREGATE, cleanup, commit, makeFixtureRepo, type FixtureRepo } from './helpers/pregate-fixture';
|
||||
|
||||
const repos: FixtureRepo[] = [];
|
||||
afterEach(() => { while (repos.length) cleanup(repos.pop()!); });
|
||||
|
||||
const LANE = 'pregate.json/remote_gate';
|
||||
const NOW = new Date('2026-11-02T12:00:00Z');
|
||||
const LOCAL_OS = process.platform === 'darwin' ? 'macOS' : 'Linux';
|
||||
const PROFILE: RunProfile = { workflow: 'free-tests.yml', events: ['pull_request', 'workflow_dispatch'], jobs: ['unit'], artifact: 'tested-tree', os: LOCAL_OS };
|
||||
const PAID = 'test/nightly/retry-heavy.test.js';
|
||||
const FREE = ['test/retry.test.js', 'test/util.test.js'];
|
||||
|
||||
function fixture(): FixtureRepo {
|
||||
const repo = makeFixtureRepo({ pregate: { ...BASE_PREGATE, remote_gate: 'node scripts/remote-gate.js' } });
|
||||
repos.push(repo);
|
||||
commit(repo, { 'lib/util.js': 'exports.add = (a, b) => a + b + 0;\n', 'lib/retry.js': 'exports.MAX_RETRIES = 3;\nexports.TIMEOUT_MS = 5000;\nexports.retry = (fn) => fn();\nexports.touched = true;\n' }, 'touch two subjects');
|
||||
return repo;
|
||||
}
|
||||
|
||||
/** Run 100 of free-tests.yml at HEAD, green, its tree.json listing `files`; the gate's inventory is FREE and its skipped command names PAID. */
|
||||
function runCheck(repo: FixtureRepo, files: string[]): RunCheck {
|
||||
const head = repo.git('rev-parse', 'HEAD');
|
||||
const api: ApiRun = { id: 100, run_attempt: 1, status: 'completed', conclusion: 'success', event: 'workflow_dispatch', head_sha: head, path: '.github/workflows/free-tests.yml', created_at: '2026-11-02T10:00:00Z', updated_at: '2026-11-02T10:30:00Z', repository: { full_name: 'fx/fx' }, head_repository: { full_name: 'fx/fx' } };
|
||||
return {
|
||||
repo: 'fx/fx', profiles: { 'ci-reuse/free-tests': PROFILE }, now: NOW,
|
||||
gateRuns: { profiles: ['ci-reuse/free-tests'], inventory: 'INVENTORY', skipped: 'SKIPPED' },
|
||||
inventory: (command, _tree, selected) => (command === 'SKIPPED' ? (selected ?? []).filter(f => f === PAID) : FREE),
|
||||
deps: {
|
||||
run: () => api, runsForHead: (_wf, sha) => (sha === head ? [api] : []), jobs: () => [{ name: 'unit', conclusion: 'success' }],
|
||||
artifact: () => ({ run_id: 100, run_attempt: 1, event: 'workflow_dispatch', sha: head, head_sha: head, tree: repo.git('rev-parse', 'HEAD^{tree}'), os: [LOCAL_OS], files, tests: files.length * 4 }),
|
||||
commitTree: sha => repo.git('rev-parse', `${sha}^{tree}`),
|
||||
},
|
||||
} as RunCheck;
|
||||
}
|
||||
|
||||
async function carried(repo: FixtureRepo, check: RunCheck): Promise<{ file: PregateFile; progress: string[] }> {
|
||||
const progress: string[] = [];
|
||||
const loaded = buildContext({ repoRoot: repo.work, base: 'main', policyFrom: 'base', allowRepoCommands: true, env: { ...process.env, ...repo.env }, progress: l => progress.push(l), cores: 4, testBackend: 'local' });
|
||||
if (!('ctx' in loaded)) throw new Error('fixture context refused');
|
||||
loaded.ctx.fromRun = { ref: '100', check };
|
||||
return { file: await runPregate(loaded.ctx, { stage: 'tests' }), progress };
|
||||
}
|
||||
const gate = (file: PregateFile) => file.requires_remote?.find(r => r.lane === LANE);
|
||||
|
||||
describe.skipIf(process.platform === 'win32')('CI carry clears the remote gate on a small machine', () => {
|
||||
// Value: protects=a carried run whose remainder is only gate-skipped (paid) files leaves no obligation nothing can clear;
|
||||
// fails_when=the remainder stays a remote_gate obligation that the gate reports tests=0 for and clear --run refuses paid_only;
|
||||
// why_new=GSTA-258 A, the PREGATE_REMOTE_PAID_ONLY dead end seen at 0fba5533 (112 paid files after run 41554380563); seam=runPregate with a fake run API
|
||||
test('the carried run clears the gate when only skipped files remain, and the receipt counts them', async () => {
|
||||
const repo = fixture();
|
||||
const check = runCheck(repo, FREE);
|
||||
const { file, progress } = await carried(repo, check);
|
||||
const obligation = gate(file)!;
|
||||
expect(obligation.tests).toEqual([PAID]);
|
||||
expect(clearDecision({ ...file, requires_remote: [{ ...obligation, receipt: null }] }, LANE, { runs: ['100'], evidence: check }, repo.work)).toMatchObject({ refused: 'PREGATE_RECEIPT_INVALID', message: expect.stringContaining('reason=paid_only') });
|
||||
expect([obligation.receipt, obligation.cleared_by]).toEqual([expect.stringMatching(new RegExp(`^remote-gate tree=${file.tree.slice(0, 12)} tests=8 skipped=1 runs=100/1 `)), 'ci-carry']);
|
||||
expect(progress).toContain(`CI_CARRY: cleared ${LANE} ${obligation.receipt}`);
|
||||
expect(file.verdict).toBe('requires-remote cleared');
|
||||
}, 120_000);
|
||||
|
||||
// Value: protects=the carry clears the gate only under clear --run's own checks; fails_when=a run that misses an inventory file clears it
|
||||
test('a run that misses a gate inventory file leaves the gate uncleared and says why', async () => {
|
||||
const repo = fixture();
|
||||
const { file, progress } = await carried(repo, runCheck(repo, ['test/util.test.js']));
|
||||
expect(gate(file)?.receipt).toBeNull();
|
||||
expect(progress).toContainEqual(expect.stringMatching(new RegExp(`^CI_CARRY: ${LANE} stays uncleared \\(REUSE: refused reason=inventory_uncovered`)));
|
||||
}, 120_000);
|
||||
|
||||
// Value: protects=when the base declares remote_gate_runs, the uncleared gate names the CI route first and the VM only as an opt-in;
|
||||
// fails_when=the obligation sends an agent to the paid VM while a CI run at the same tree would clear it; why_new=GSTA-258 A, plan choice F1
|
||||
test('with remote_gate_runs on the base, an uncleared gate names the CI route first and the VM as the opt-in', async () => {
|
||||
const repo = makeFixtureRepo({ pregate: { ...BASE_PREGATE, remote_gate: 'node scripts/remote-gate.js', run_profiles: { 'ci-reuse/free-tests': PROFILE }, remote_gate_runs: { profiles: ['ci-reuse/free-tests'], inventory: 'echo test/retry.test.js' } } });
|
||||
repos.push(repo);
|
||||
commit(repo, { 'lib/retry.js': 'exports.MAX_RETRIES = 3;\nexports.TIMEOUT_MS = 5000;\nexports.retry = (fn) => fn();\nexports.touched = 2;\n' }, 'touch retry');
|
||||
const loaded = buildContext({ repoRoot: repo.work, base: 'main', policyFrom: 'base', allowRepoCommands: true, env: { ...process.env, ...repo.env }, progress: () => {}, cores: 4, testBackend: 'local' });
|
||||
if (!('ctx' in loaded)) throw new Error('fixture context refused');
|
||||
const file = await runPregate(loaded.ctx, { stage: 'tests' });
|
||||
expect(gate(file)?.receipt).toBeNull();
|
||||
const detail = file.checks.find(c => c.id === 'lanes')?.detail ?? '';
|
||||
expect(detail).toContain(`a verified CI run at this tree clears it: push, then gstack-ci-carry wait --base main and rerun gstack-pregate --allow-repo-commands`);
|
||||
expect(detail.indexOf('gstack-ci-carry wait')).toBeLessThan(detail.indexOf('opt in to the VM: gstack-pregate --remote --spend-cap <usd> --allow-repo-commands'));
|
||||
loaded.ctx.ciCarry = false;
|
||||
const off = (await runPregate(loaded.ctx, { stage: 'tests' })).checks.find(c => c.id === 'lanes')?.detail ?? '';
|
||||
expect(off).not.toContain('gstack-ci-carry wait');
|
||||
expect(off).toContain('run it, then gstack-pregate clear --remote pregate.json/remote_gate');
|
||||
}, 120_000);
|
||||
});
|
||||
@@ -50,12 +50,12 @@ describe('normalizer v2 as collected: Blocks: yes is CRITICAL, Blocks: no never
|
||||
// Value: protects=the rule round 5's G1 was scored under, on the path production runs (gstack-review-dispatch collect);
|
||||
// fails_when=Blocks: yes collects below CRITICAL or Blocks: no lowers a label; seam=parseFindings
|
||||
test.each([
|
||||
['FIXABLE: race in a.ts:3\nSeverity: Medium\nBlocks: no', 'CRITICAL'],
|
||||
['FIXABLE: race in a.ts:3\nSeverity: Medium\nBlocks: no', 'Medium'],
|
||||
['INVESTIGATE: odd lock in b.ts:9\nSeverity: Low\nBlocks: yes', 'CRITICAL'],
|
||||
['INVESTIGATE: odd lock in b.ts:9\nBlocks: no', 'INVESTIGATE'],
|
||||
['FIXABLE: race in a.ts:3. Severity: Medium. Blocks: no', 'CRITICAL'],
|
||||
['1. FIXABLE: race in a.ts:3 — Severity: Low, Blocks: no', 'CRITICAL'],
|
||||
['- FIXABLE: race in a.ts:3\n- Severity: Medium\n- Blocks: no', 'CRITICAL'],
|
||||
['FIXABLE: race in a.ts:3. Severity: Medium. Blocks: no', 'Medium'],
|
||||
['1. FIXABLE: race in a.ts:3 — Severity: Low, Blocks: no', 'Low'],
|
||||
['- FIXABLE: race in a.ts:3\n- Severity: Medium\n- Blocks: no', 'Medium'],
|
||||
] as const)('native %j -> %s', (output, want) => expect(parseFindings('adversarial-native', output).map(g => g.label)).toEqual([want]));
|
||||
test.each([
|
||||
['1. Drops the last row (q.ts:7). Severity: Critical. Blocks: no', 'Critical'],
|
||||
|
||||
@@ -0,0 +1,104 @@
|
||||
import { afterAll, expect, test } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
|
||||
import { RESOLVERS } from '../scripts/resolvers';
|
||||
import { HOST_PATHS, type TemplateContext } from '../scripts/resolvers/types';
|
||||
|
||||
const BIN = path.resolve(import.meta.dir, '../bin');
|
||||
const ROOT = fs.mkdtempSync(path.join(fs.realpathSync(os.tmpdir()), 'qa-report-dir-'));
|
||||
afterAll(() => fs.rmSync(ROOT, { recursive: true, force: true }));
|
||||
|
||||
const IDENTITY = { GIT_AUTHOR_NAME: 'fixture', GIT_AUTHOR_EMAIL: 'fixture@example.com', GIT_COMMITTER_NAME: 'fixture', GIT_COMMITTER_EMAIL: 'fixture@example.com' };
|
||||
|
||||
function product() {
|
||||
const root = fs.mkdtempSync(path.join(ROOT, 'product-'));
|
||||
const git = (...args: string[]) => {
|
||||
const r = spawnSync('git', args, { cwd: root, encoding: 'utf8', timeout: 10_000, env: { ...process.env, ...IDENTITY } });
|
||||
expect(r.status, r.stderr).toBe(0);
|
||||
return r.stdout;
|
||||
};
|
||||
git('init', '-q');
|
||||
fs.writeFileSync(path.join(root, 'app.ts'), 'export const x = 1;\n');
|
||||
git('add', 'app.ts');
|
||||
git('commit', '-q', '-m', 'seed');
|
||||
const tool = (name: string, ...args: string[]) => {
|
||||
const r = spawnSync(process.execPath, [path.join(BIN, name), ...args], { cwd: root, encoding: 'utf8', timeout: 15_000 });
|
||||
expect(r.error, r.error?.message).toBeUndefined();
|
||||
return r;
|
||||
};
|
||||
const tree = () => git('status', '--porcelain', '--untracked-files=all', '--ignored=no') + git('ls-files', '--others', '--exclude-standard');
|
||||
return { root, tool, tree };
|
||||
}
|
||||
|
||||
function reportDir(f: ReturnType<typeof product>): string {
|
||||
const r = f.tool('gstack-qa-evidence', 'report-dir');
|
||||
expect(r.status, r.stderr).toBe(0);
|
||||
return JSON.parse(r.stdout.trim().slice('QA_EVIDENCE '.length)).reportDir;
|
||||
}
|
||||
|
||||
test('the /ship QA report root leaves a product tree without a .gstack ignore rule unchanged', () => {
|
||||
const f = product();
|
||||
expect(f.tree()).toBe('');
|
||||
const dir = reportDir(f);
|
||||
expect(path.relative(f.root, dir).split(path.sep).slice(0, 3)).toEqual(['.gstack', 'tmp', 'qa-reports']);
|
||||
const deadline = path.join(dir, 'deadline.json');
|
||||
expect(f.tool('gstack-qa-deadline', 'start', deadline, '300').status).toBe(0);
|
||||
const capture = f.tool('gstack-qa-evidence', 'capture', dir, '001', '--public', '--deadline', deadline, '--', process.execPath, '-e', 'console.log(JSON.stringify({ ok: true }))');
|
||||
expect(capture.status, capture.stderr).toBe(0);
|
||||
fs.writeFileSync(path.join(dir, 'report.md'), '## Exploratory QA\n');
|
||||
expect(f.tree()).toBe('');
|
||||
});
|
||||
|
||||
test('each report-dir call owns a fresh run so a second QA run never overwrites the first', () => {
|
||||
const f = product();
|
||||
const first = reportDir(f);
|
||||
const second = reportDir(f);
|
||||
expect(second).not.toBe(first);
|
||||
expect(fs.readdirSync(first)).toEqual([]);
|
||||
expect(fs.readdirSync(second)).toEqual([]);
|
||||
});
|
||||
|
||||
test('report-dir refuses a linked .gstack instead of creating directories through it', () => {
|
||||
const f = product();
|
||||
const elsewhere = fs.mkdtempSync(path.join(ROOT, 'elsewhere-'));
|
||||
fs.symlinkSync(elsewhere, path.join(f.root, '.gstack'));
|
||||
const r = f.tool('gstack-qa-evidence', 'report-dir');
|
||||
expect(r.status).toBe(2);
|
||||
expect(r.stderr).toContain('report-dir refuses the linked .gstack');
|
||||
expect(fs.readdirSync(elsewhere)).toEqual([]);
|
||||
});
|
||||
|
||||
test('report-dir refuses when a product .gitignore un-ignores .gstack/tmp', () => {
|
||||
const f = product();
|
||||
fs.writeFileSync(path.join(f.root, '.gitignore'), '!/.gstack/tmp/\n!/.gstack/tmp/**\n');
|
||||
const r = f.tool('gstack-qa-evidence', 'report-dir');
|
||||
expect(r.status).toBe(2);
|
||||
expect(r.stderr).toContain('.gstack/tmp is not git-ignored here');
|
||||
});
|
||||
|
||||
test('report-dir names the filesystem cause when .gstack is not a directory', () => {
|
||||
const f = product();
|
||||
fs.writeFileSync(path.join(f.root, '.gstack'), 'not a directory\n');
|
||||
const r = f.tool('gstack-qa-evidence', 'report-dir');
|
||||
expect(r.status).toBe(2);
|
||||
expect(r.stderr).toContain('report-dir failed: ENOTDIR');
|
||||
});
|
||||
|
||||
test('report-dir refuses outside a git work tree instead of guessing a location', () => {
|
||||
const outside = fs.mkdtempSync(path.join(ROOT, 'plain-'));
|
||||
const r = spawnSync(process.execPath, [path.join(BIN, 'gstack-qa-evidence'), 'report-dir'], { cwd: outside, encoding: 'utf8', timeout: 15_000, env: { ...process.env, GIT_CEILING_DIRECTORIES: ROOT } });
|
||||
expect(r.status).toBe(2);
|
||||
expect(r.stderr).toContain('report-dir needs a git work tree');
|
||||
expect(fs.readdirSync(outside)).toEqual([]);
|
||||
});
|
||||
|
||||
test('caller QA without a resolved REPORT_DIR (/review, /ship) falls back to report-dir, not the product tree', () => {
|
||||
const line = 'Reuse resolved REPORT_DIR; otherwise use the `reportDir` `EVIDENCE_TOOL report-dir` prints.';
|
||||
const ctx: TemplateContext = { host: 'claude', skillName: 'qa', tmplPath: '', paths: HOST_PATHS.claude };
|
||||
expect(RESOLVERS.QA_EXPLORATORY(ctx)).toContain(line);
|
||||
expect(fs.readFileSync(path.resolve(import.meta.dir, '../qa/sections/exploratory.md'), 'utf8')).toContain(line);
|
||||
expect(fs.readFileSync(path.resolve(import.meta.dir, '../qa/SKILL.md.tmpl'), 'utf8')).toContain('Use the directory as `REPORT_DIR`');
|
||||
});
|
||||
@@ -329,4 +329,106 @@ describe('a branch-added row left at the fork version after the base moved (GST-
|
||||
expect(receipt.out).toContain("id: 'base_row'");
|
||||
expect(receipt.out).not.toContain("id: 'mine'");
|
||||
});
|
||||
|
||||
// Value: protects=a base row whose text a branch row duplicates keeps the old-version floor in the tree receipt
|
||||
// (GST-979, the #3862 outside voice); fails_when=the side matches by text, so a hand move of the base copy reads as
|
||||
// same-modulo-stamps; seam=lib/stamp-only.ts branchSides positions
|
||||
test('a base row whose text the branch duplicates still refuses a hand move; the branch copy\'s own move passes (GST-979)', () => {
|
||||
const repo = makeFixtureRepo({ prefix: 'restamp-dup-row-', policy: receiptPolicy(true) }); repos.push(repo);
|
||||
write(repo, NOTICE, notice([row('1.2.3.0', 'base_row')]));
|
||||
repo.git('add', '-A'); repo.git('commit', '-qm', 'notice rows'); repo.git('push', '-q', 'origin', 'main');
|
||||
makePr(repo, 'pr-dup');
|
||||
expect(restamp(repo, ['--version', '1.2.4.0']).status).toBe(EXIT.ok);
|
||||
repo.git('commit', '-qam', 'stamp 1.2.4.0');
|
||||
write(repo, NOTICE, notice([row('1.2.3.0', 'base_row'), '', '', row('1.2.3.0', 'base_row')]));
|
||||
repo.git('commit', '-qam', 'dup: the same row text at the fork version');
|
||||
const gated = repo.git('rev-parse', 'HEAD');
|
||||
expect(restamp(repo, ['--version', '1.2.5.0']).status).toBe(EXIT.ok);
|
||||
expect(readFile(repo, NOTICE)).toBe(notice([row('1.2.3.0', 'base_row'), '', '', row('1.2.5.0', 'base_row')]));
|
||||
repo.git('commit', '-qam', 'stamp dup');
|
||||
const own = runBin(repo, 'gstack-tree-receipt', ['--gated', gated, '--base', 'main']);
|
||||
expect(own.out, own.err).toContain('TREE: same-modulo-stamps');
|
||||
write(repo, NOTICE, notice([row('1.2.5.0', 'base_row'), '', '', row('1.2.5.0', 'base_row')]));
|
||||
repo.git('commit', '-qam', 'move the base copy by hand');
|
||||
const hand = runBin(repo, 'gstack-tree-receipt', ['--gated', gated, '--base', 'main']);
|
||||
expect(hand.out, hand.err).toContain('TREE: changed');
|
||||
});
|
||||
|
||||
// Value: protects=the receipt carry follows a stamp that moves the branch's own fork-version row, the move the writer
|
||||
// makes since GST-963 (GST-980); fails_when=the carry keeps the branch-added row's bytes in both product patch-ids and
|
||||
// refuses RECEIPT_CARRY_PRODUCT_CHANGED; seam=bin/gstack-ship-receipt carry walk
|
||||
test('the carry follows a stamp that moves the branch\'s own fork-version row, and still refuses a product edit beside it (GST-980)', () => {
|
||||
const repo = makeFixtureRepo({ prefix: 'restamp-carry-row-', policy: policy(true) }); repos.push(repo);
|
||||
write(repo, NOTICE, notice([row('1.2.3.0', 'base_row')]));
|
||||
fs.rmSync(path.join(repo.work, 'package-lock.json'));
|
||||
repo.git('add', '-A'); repo.git('commit', '-qm', 'notice rows, no lockfile'); repo.git('push', '-q', 'origin', 'main');
|
||||
makePr(repo, 'pr-carry');
|
||||
expect(restamp(repo, ['--version', '1.2.4.0']).status).toBe(EXIT.ok);
|
||||
repo.git('commit', '-qam', 'stamp 1.2.4.0');
|
||||
write(repo, NOTICE, notice([row('1.2.3.0', 'base_row'), '', row('1.2.3.0', 'mine')]));
|
||||
repo.git('commit', '-qam', 'mine: claims 1.2.4.0 with a row at the fork version');
|
||||
const body = path.join(repo.root, 'body.md');
|
||||
fs.writeFileSync(body, '## Summary\n');
|
||||
const w = runBin(repo, 'gstack-ship-receipt', ['write', '--pr', '3', '--base', 'main', '--session-kind', 'x', '--body', body]);
|
||||
expect(w.status, w.err).toBe(EXIT.ok);
|
||||
expect(restamp(repo, ['--version', '1.2.5.0']).status).toBe(EXIT.ok);
|
||||
expect(readFile(repo, NOTICE)).toBe(notice([row('1.2.3.0', 'base_row'), '', row('1.2.5.0', 'mine')]));
|
||||
repo.git('commit', '-qam', 'stamp 1.2.5.0');
|
||||
const carried = runBin(repo, 'gstack-ship-receipt', ['carry', '--body', body, '--base', 'main']);
|
||||
expect(carried.err).not.toContain('RECEIPT_CARRY_PRODUCT_CHANGED');
|
||||
expect(carried.out, carried.err).toContain('CARRY: carried');
|
||||
write(repo, 'src.ts', 'export const x = 1;\n');
|
||||
repo.git('add', '-A'); repo.git('commit', '-qm', 'a product change after the stamp');
|
||||
const changed = runBin(repo, 'gstack-ship-receipt', ['carry', '--body', body, '--base', 'main']);
|
||||
expect(changed.status).toBe(EXIT.refused);
|
||||
expect(changed.err).toContain('RECEIPT_CARRY_PRODUCT_CHANGED');
|
||||
});
|
||||
|
||||
/** The GST-980 carry state: a receipt written at the claim 1.2.4.0 with the branch's own row, then the stamp to 1.2.5.0, uncommitted. */
|
||||
function carryAfterStamp(prefix: string, baseRows: string[], mine: string, stamped: string[]): { repo: FixtureRepo; body: string } {
|
||||
const repo = makeFixtureRepo({ prefix, policy: policy(true) }); repos.push(repo);
|
||||
write(repo, NOTICE, notice(baseRows));
|
||||
fs.rmSync(path.join(repo.work, 'package-lock.json'));
|
||||
repo.git('add', '-A'); repo.git('commit', '-qm', 'notice rows, no lockfile'); repo.git('push', '-q', 'origin', 'main');
|
||||
makePr(repo, 'pr-carry');
|
||||
expect(restamp(repo, ['--version', '1.2.4.0']).status).toBe(EXIT.ok);
|
||||
repo.git('commit', '-qam', 'stamp 1.2.4.0');
|
||||
write(repo, NOTICE, notice([...baseRows, '', mine]));
|
||||
repo.git('commit', '-qam', 'mine: a row of its own');
|
||||
const body = path.join(repo.root, 'body.md');
|
||||
fs.writeFileSync(body, '## Summary\n');
|
||||
const w = runBin(repo, 'gstack-ship-receipt', ['write', '--pr', '3', '--base', 'main', '--session-kind', 'x', '--body', body]);
|
||||
expect(w.status, w.err).toBe(EXIT.ok);
|
||||
expect(restamp(repo, ['--version', '1.2.5.0']).status).toBe(EXIT.ok);
|
||||
expect(readFile(repo, NOTICE)).toBe(notice(stamped));
|
||||
return { repo, body };
|
||||
}
|
||||
|
||||
// Value: protects=the carry leaves out only a stamp on the branch's own rows, never a version edit on a base line of
|
||||
// the same file (GST-980 review); fails_when=the carry trusts the whole-text version swap, so a hand move of a base
|
||||
// line's version mention rides along with the row stamp; seam=bin/gstack-ship-receipt productSpec ownRows
|
||||
test('the carry refuses a hand version edit on a base line of a stamp file whose own row it follows (GST-980)', () => {
|
||||
const min = "export const MIN = '1.2.4.0';";
|
||||
const { repo, body } = carryAfterStamp('restamp-carry-base-line-', [row('1.2.3.0', 'base_row'), min], row('1.2.4.0', 'mine'),
|
||||
[row('1.2.3.0', 'base_row'), min, '', row('1.2.5.0', 'mine')]);
|
||||
write(repo, NOTICE, notice([row('1.2.3.0', 'base_row'), min.replace('1.2.4.0', '1.2.5.0'), '', row('1.2.5.0', 'mine')]));
|
||||
repo.git('commit', '-qam', 'stamp 1.2.5.0, and move the base pin by hand');
|
||||
const carried = runBin(repo, 'gstack-ship-receipt', ['carry', '--body', body, '--base', 'main']);
|
||||
expect(carried.status, carried.out).toBe(EXIT.refused);
|
||||
expect(carried.err).toContain('RECEIPT_CARRY_PRODUCT_CHANGED');
|
||||
});
|
||||
|
||||
// Value: protects=a mode change rides on no stamp: the carry leaves out a stamp file only when its mode is the
|
||||
// reviewed one (GST-980 review); fails_when=the left-out stamp file drops its mode line from both patch-ids;
|
||||
// seam=bin/gstack-ship-receipt productSpec ownRows
|
||||
test('the carry refuses a mode change on a stamp file whose own row it follows (GST-980)', () => {
|
||||
const { repo, body } = carryAfterStamp('restamp-carry-mode-', [row('1.2.3.0', 'base_row')], row('1.2.3.0', 'mine'),
|
||||
[row('1.2.3.0', 'base_row'), '', row('1.2.5.0', 'mine')]);
|
||||
fs.chmodSync(path.join(repo.work, NOTICE), 0o755);
|
||||
repo.git('commit', '-qam', 'stamp 1.2.5.0, and make the file executable');
|
||||
expect(repo.git('ls-tree', 'HEAD', '--', NOTICE)).toStartWith('100755 ');
|
||||
const carried = runBin(repo, 'gstack-ship-receipt', ['carry', '--body', body, '--base', 'main']);
|
||||
expect(carried.status, carried.out).toBe(EXIT.refused);
|
||||
expect(carried.err).toContain('RECEIPT_CARRY_PRODUCT_CHANGED');
|
||||
});
|
||||
});
|
||||
@@ -440,7 +440,10 @@ describe('gstack-review-dispatch sent, collect, judge and verify: nothing change
|
||||
['CRITICAL: x.ts:4 drops rows (pass 1 had Severity: Low)', [['CRITICAL', 'CRITICAL: x.ts:4 drops rows (pass 1 had Severity: Low)']]],
|
||||
])('fix-check shape %j', (output, want) => expect(parseFindings('outside-voice', output).map(f => [f.label, f.raw])).toEqual(want));
|
||||
test('the native voice\'s FIXABLE collects as CRITICAL under its role with or without a dispatch name', () => {
|
||||
for (const reader of ['adversarial-native', 'adversarial-native/adversarial']) expect(parseFindings(reader, 'FIXABLE: race in a.ts:3\nSeverity: Medium\nBlocks: no').map(f => f.label)).toEqual(['CRITICAL']);
|
||||
for (const reader of ['adversarial-native', 'adversarial-native/adversarial']) {
|
||||
expect(parseFindings(reader, 'FIXABLE: race in a.ts:3\nSeverity: Medium').map(f => f.label)).toEqual(['CRITICAL']);
|
||||
expect(parseFindings(reader, 'FIXABLE: race in a.ts:3\nSeverity: Medium\nBlocks: no').map(f => f.label)).toEqual(['Medium']);
|
||||
}
|
||||
expect(parseFindings('outside-voice', 'FIXABLE: race in a.ts:3\nSeverity: Medium').map(f => f.label)).toEqual(['Medium']);
|
||||
});
|
||||
test('a non-prefix Blocks: yes under a labeled finding raises it to CRITICAL', () => {
|
||||
@@ -529,7 +532,7 @@ describe('gstack-review-dispatch sent, collect, judge and verify: nothing change
|
||||
test('unlabeled detail bullets and the Severity line after them stay on the open paragraph finding', () => {
|
||||
const native = fs.readFileSync(path.join(ROOT, 'test/fixtures/review-ledger/gsta-196-native.txt'), 'utf8');
|
||||
expect(parseFindings('adversarial-native', native).map(f => [f.label, f.raw.slice(0, 12), f.raw.endsWith('| Severity: ' + (f.label === 'High' ? 'High' : 'Low') + '. Blocks: no.')]))
|
||||
.toEqual([['High', '**1. A PR in', true], ['CRITICAL', '**2. A case ', true], ['CRITICAL', '**3. Local `', true], ['Low', '**4. No trai', true]]);
|
||||
.toEqual([['High', '**1. A PR in', true], ['Low', '**2. A case ', true], ['Low', '**3. Local `', true], ['Low', '**4. No trai', true]]);
|
||||
expect(parseFindings('outside-voice', '1. FIXABLE: race in a.ts:3\n- Unlabeled sibling at b.ts:9\nSeverity: Medium').map(f => f.label)).toEqual(['FIXABLE', 'Medium']);
|
||||
expect(parseFindings('outside-voice', '**Retry storm in r.ts:2 (INVESTIGATE).**\n- detail\n- High: a second defect in s.ts:4\nSeverity: Medium').map(f => f.label)).toEqual(['INVESTIGATE', 'High']);
|
||||
expect(parseFindings('outside-voice', 'Low: cache TTL mismatch (cache.ts:4)\n\n2. Token never expires (auth.ts:9)\nSeverity: High\nBlocks: yes').map(f => [f.label, f.raw]))
|
||||
@@ -548,7 +551,7 @@ describe('gstack-review-dispatch sent, collect, judge and verify: nothing change
|
||||
const native = fs.readFileSync(path.join(ROOT, 'test/fixtures/review-ledger/gsta-196-native.txt'), 'utf8');
|
||||
const r = collect(path.join(d, 'l.jsonl'), 'adversarial-native', native, path.join(d, 'm.jsonl'));
|
||||
expect(r.lines.map(l => [l.label, l.stated])).toEqual([
|
||||
['High', { severity: 'High', blocks: 'no' }], ['CRITICAL', { severity: 'Low', blocks: 'no' }], ['CRITICAL', { severity: 'Low', blocks: 'no' }], ['Low', { severity: 'Low', blocks: 'no' }]]);
|
||||
['High', { severity: 'High', blocks: 'no' }], ['Low', { severity: 'Low', blocks: 'no' }], ['Low', { severity: 'Low', blocks: 'no' }], ['Low', { severity: 'Low', blocks: 'no' }]]);
|
||||
expect(fs.readFileSync(path.join(d, 'm.jsonl'), 'utf8').trim().split('\n').map(l => JSON.parse(l).stated)).toEqual(r.lines.map(l => l.stated));
|
||||
const json = collect(path.join(d, 'l2.jsonl'), 'fix-check', '{"severity":"INFORMATIONAL","path":"a.ts","line":3,"summary":"Severity: High is the wrong word here"}\n', path.join(d, 'm2.jsonl'));
|
||||
expect(json.lines.map(l => [l.label, l.stated])).toEqual([['INFORMATIONAL', undefined]]);
|
||||
@@ -584,7 +587,8 @@ describe('gstack-review-dispatch sent, collect, judge and verify: nothing change
|
||||
for (const [line, label] of kept) expect([line, parseFindings('outside-voice', line).map(f => f.label)]).toEqual([line, [label]]);
|
||||
const native = fs.readFileSync(path.join(ROOT, 'test/fixtures/review-ledger/harn15-152-p1-adversarial-native.txt'), 'utf8');
|
||||
expect(parseFindings('adversarial-native', native).map(f => f.label)).toEqual(['CRITICAL', 'INVESTIGATE', 'INVESTIGATE', 'CRITICAL', 'INVESTIGATE', 'INVESTIGATE', 'CRITICAL']);
|
||||
expect(parseFindings('adversarial-native', '**1. Race in a.ts:3 (`FIXABLE`).**\nSeverity: Low. Blocks: no').map(f => f.label)).toEqual(['CRITICAL']);
|
||||
expect(parseFindings('adversarial-native', '**1. Race in a.ts:3 (`FIXABLE`).**\nSeverity: Low. Blocks: no').map(f => f.label)).toEqual(['Low']);
|
||||
expect(parseFindings('adversarial-native', '**1. Race in a.ts:3 (`FIXABLE`).**\nSeverity: Low.').map(f => f.label)).toEqual(['CRITICAL']);
|
||||
expect(parseFindings('red-team', '1. Race in a.ts:3 (FIXABLE).').map(f => f.label)).toEqual(['FIXABLE']);
|
||||
expect(parseFindings('adversarial-native', 'No rate limit is FIXABLE (login.ts:9)').map(f => f.label)).toEqual(['CRITICAL']);
|
||||
});
|
||||
@@ -626,7 +630,7 @@ describe('gstack-review-dispatch sent, collect, judge and verify: nothing change
|
||||
test('item layouts collect as on main; the later-paragraph split and indented label headings are known deferred shapes', () => {
|
||||
const native = fs.readFileSync(path.join(ROOT, 'test/fixtures/review-ledger/harn15-ship2-p2-adversarial-native.txt'), 'utf8');
|
||||
const found = parseFindings('adversarial-native', native);
|
||||
expect(found.map(f => f.label)).toEqual(['CRITICAL', 'CRITICAL', 'CRITICAL', 'CRITICAL', 'CRITICAL', 'Low']);
|
||||
expect(found.map(f => f.label)).toEqual(['Medium', 'Low', 'Low', 'Low', 'Low', 'Low']);
|
||||
expect(parseFindings('outside-voice', '1. **Race on the lock** (a.ts:3)\n - the writer holds no lock\n\n Fix: take it first. Severity: High. Blocks: no.').map(f => f.label)).toEqual(['High']);
|
||||
expect(parseFindings('outside-voice', '1. Lock race (a.ts:3)\n\n Fix: take it first. Blocks: yes\n2. Cache drift (c.ts:4)').map(f => f.label)).toEqual(['CRITICAL']);
|
||||
expect(parseFindings('red-team', '1. Unescaped query\n {"severity":"critical","summary":"Low-level race in q.ts"}').map(f => [f.label, f.raw]))
|
||||
|
||||
@@ -18,7 +18,7 @@ describe('native FIXABLE by review class (GST-987)', () => {
|
||||
// no_claim=which class a diff belongs to (lib/review-depth.ts decides)
|
||||
test('#141 native #4 and #6 collect at their own Severity on a code-class plan', () => {
|
||||
expect(parseFindings('adversarial-native', NATIVE, fixableRaises(CODE)).map(f => f.label)).toEqual(['Medium', 'Low']);
|
||||
expect(parseFindings('adversarial-native', NATIVE).map(f => f.label)).toEqual(['CRITICAL', 'CRITICAL']);
|
||||
expect(parseFindings('adversarial-native', NATIVE).map(f => f.label)).toEqual(['Medium', 'Low']);
|
||||
});
|
||||
test('gate class, any risk reason, a depth error or an unreadable plan keeps normalizer v2', () => {
|
||||
for (const lines of [plan('gate', 'gate-class,runtime=9'), plan('code', 'risk:money,runtime=9'), plan('code', 'risk:data,runtime=9'), plan('code', 'error:DIFF,runtime=9'), plan('code', 'runtime=9,history:fixed-blocking'), plan('code', 'runtime=9,floor:full'), plan('code', ''), plan('unknown', 'runtime=9'), [CODE[0]!], [], undefined])
|
||||
@@ -47,7 +47,7 @@ describe('native FIXABLE by review class (GST-987)', () => {
|
||||
['### 1. Race in a.ts:3 — FIXABLE\n\nBody.\n\n**Severity:** Medium\n**Blocks:** no\n\n### 2. Drift in b.ts:9 — FIXABLE\n\nBody.\n\n**Severity:** Low\n**Blocks:** no'],
|
||||
])('a numbered finding heading keeps its body paragraph and fields (#149) %#', (output) => {
|
||||
expect(parseFindings('adversarial-native', output, false).map(f => f.label)).toEqual(['Medium', 'Low']);
|
||||
expect(parseFindings('adversarial-native', output, true).map(f => f.label)).toEqual(['CRITICAL', 'CRITICAL']);
|
||||
expect(parseFindings('adversarial-native', output, true).map(f => f.label)).toEqual(['Medium', 'Low']);
|
||||
});
|
||||
test('a labeled or located paragraph after a numbered heading stays its own finding (#149 joins only plain body text)', () => {
|
||||
for (const raises of [true, false]) {
|
||||
@@ -74,9 +74,9 @@ describe('native FIXABLE by review class (GST-987)', () => {
|
||||
};
|
||||
expect(labels(['--manifest', manifest])).toEqual(['Medium', 'Low']);
|
||||
expect(fs.readFileSync(ledger, 'utf8')).toContain('"fixable_raises":false');
|
||||
expect(labels([])).toEqual(['CRITICAL', 'CRITICAL']);
|
||||
expect(labels([])).toEqual(['Medium', 'Low']);
|
||||
expect(fs.readFileSync(ledger, 'utf8')).not.toContain('fixable_raises');
|
||||
expect(labels(['--manifest', foreign])).toEqual(['CRITICAL', 'CRITICAL']);
|
||||
expect(labels(['--manifest', foreign])).toEqual(['Medium', 'Low']);
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
});
|
||||
test('verify re-derives a code-class collect with the class it was collected under', () => {
|
||||
|
||||
@@ -0,0 +1,145 @@
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { promptHash } from '../lib/review-brief';
|
||||
import { collect, judge, ledgerBlockers, parseFindings, readJsonl, recordJudgment, summaryCandidate, verify } from '../lib/review-ledger';
|
||||
import { isBlocking, summarizeReviewLoop } from '../lib/review-scope';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const fixture = (name: string) => fs.readFileSync(path.join(ROOT, 'test/fixtures/review-ledger', name), 'utf8');
|
||||
const labels = (reader: string, output: string, raises: boolean) => parseFindings(reader, output, raises).map(f => f.label);
|
||||
const HEAD = '**1. The family check refuses two models in one call (check.ts:165-169). FIXABLE.**';
|
||||
|
||||
describe('one record per native finding at its stated severity (GST-1031 case 1, ruling A)', () => {
|
||||
// Value: protects=a native finding whose reader states Severity and Blocks: no collects once, at that severity, on every class, and a
|
||||
// heading keeps its body (located or numbered bullets, a bold subheading, a located paragraph) until its fields arrive;
|
||||
// fails_when=the heading collects CRITICAL beside a separate severity record, or the gate class raises a Blocks: no finding;
|
||||
// why_new=GEVA-196 (gstack-evals #153) and GSTA-150 GST-1027 looped Steps 9-11 at about 12M tokens a hit; seam=parseFindings;
|
||||
// no_claim=which class a diff belongs to (lib/review-depth.ts decides)
|
||||
test.each([true, false])('the GEVA-196 shape collects Medium, Low, Low with raises=%p', (raises) => {
|
||||
const found = parseFindings('adversarial-native', fixture('gst-1031-native.txt'), raises);
|
||||
expect(found.map(f => f.label)).toEqual(['Medium', 'Low', 'Low']);
|
||||
expect(found.map(f => f.raw.startsWith('**'))).toEqual([true, true, true]);
|
||||
});
|
||||
test.each([
|
||||
['located bullets', [HEAD, 'The check runs always.', '', 'Now every judge stops:', '- the default judge clashes in policy.ts:12;', '- the other judges clash in call.ts:40.', '', 'Fix: gate it on judged.', 'Severity: Medium. Blocks: no.']],
|
||||
['numbered bullets', [HEAD, 'The check runs always.', '', 'Now every judge stops:', '1. the default judge clashes;', '2. the other judges clash.', '', 'Severity: Medium. Blocks: no.']],
|
||||
['a bold subheading', [HEAD, 'The check runs always.', '', '**Why it matters**', 'So the two models cannot share a call.', 'Severity: Medium. Blocks: no.']],
|
||||
['a located paragraph', [HEAD, 'The check runs always.', '', 'The caller in call.ts:40 picks the judge by provider.', 'Severity: Medium. Blocks: no.']],
|
||||
])('a heading keeps its body with %s: one record on both classes', (_shape, lines) => {
|
||||
for (const raises of [true, false]) expect(labels('adversarial-native', (lines as string[]).join('\n'), raises)).toEqual(['Medium']);
|
||||
});
|
||||
test('negative controls: no Blocks: no, Blocks: yes, no Severity field, or a labeled body line keep the raise or their own record', () => {
|
||||
for (const raises of [true, false]) {
|
||||
expect(labels('adversarial-native', [HEAD, 'Body.', 'Severity: Medium.'].join('\n'), raises)).toEqual(['CRITICAL']);
|
||||
expect(labels('adversarial-native', [HEAD, 'Body.', 'Severity: Medium. Blocks: yes.'].join('\n'), raises)).toEqual(['CRITICAL']);
|
||||
expect(labels('adversarial-native', [HEAD, 'Body.', 'Blocks: no.'].join('\n'), raises)).toEqual(['CRITICAL']);
|
||||
expect(labels('adversarial-native', [HEAD, '', '- High: the caller in call.ts:40 leaks the key.', '', 'Severity: Medium. Blocks: no.'].join('\n'), raises)).toEqual(['CRITICAL', 'High']);
|
||||
}
|
||||
expect(labels('adversarial-native', '## FIXABLE\n1. Leak in a.ts:3\n2. Race in b.ts:4. Blocks: no', true)).toEqual(['CRITICAL', 'CRITICAL']);
|
||||
});
|
||||
test('collect records the stated severity beside the one record', () => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gst1031-'));
|
||||
const c = collect(path.join(dir, 'l.jsonl'), 'adversarial-native', fixture('gst-1031-native.txt'), path.join(dir, 'm.jsonl'));
|
||||
expect(c.lines.map(l => [l.label, l.stated])).toEqual([['Medium', { severity: 'Medium', blocks: 'no' }], ['Low', { severity: 'Low', blocks: 'no' }], ['Low', { severity: 'Low', blocks: 'no' }]]);
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
});
|
||||
});
|
||||
|
||||
describe('a closing summary is a candidate lead judgment may dismiss on record (GST-1031 case 5)', () => {
|
||||
// Value: protects=a sentence that names a severity with no file:line and no exception clause collects as a summary_candidate, which
|
||||
// judge lets lead judgment lower with a reason (recorded as dismissed) so it stops counting as a blocker, while it still collects at
|
||||
// its label; fails_when=GSTA-212's "nothing is CRITICAL" or GST-986 r2's "covers the CRITICAL fix" can only be cleared by a
|
||||
// hand-edited row, a candidate lowered without a reason passes judge, or GST-766's exception-clause line becomes a candidate;
|
||||
// why_new=both stopped /ship with NEXT: stop; seam=collect, judge, ledgerBlockers; no_claim=that a dismissal is right (it is on record)
|
||||
const dir = () => fs.mkdtempSync(path.join(os.tmpdir(), 'gst1031s-'));
|
||||
test('the negated and the plain summary sentence are candidates; dismissed with a reason they leave the blocker count', () => {
|
||||
for (const [reader, output, id] of [['fix-check', 'None of these would fail a test or let a real consumer through on input the fix was about, so nothing is CRITICAL.\n', 'fix-check#1'], ['fix-check', fixture('gst-1031-fix-check.txt'), 'fix-check#4']]) {
|
||||
const d = dir();
|
||||
const ledger = path.join(d, 'l.jsonl');
|
||||
const merged = path.join(d, 'm.jsonl');
|
||||
fs.writeFileSync(path.join(d, 'out.txt'), output!);
|
||||
const c = collect(ledger, reader!, output!, merged, path.join(d, 'out.txt'));
|
||||
expect(c.lines.filter(l => l.summary_candidate).map(l => [l.id, l.label])).toEqual([[id, 'CRITICAL']]);
|
||||
const write = (reason?: string) => { fs.writeFileSync(path.join(d, 'j.jsonl'), c.lines.map(l => JSON.stringify({ id: l.id, severity: 'INFORMATIONAL', ...(l.id === id && reason ? { reason } : {}) })).join('\n') + '\n'); return recordJudgment(ledger, c.lines, readJsonl(path.join(d, 'j.jsonl')), path.join(d, 'j.jsonl'), merged); };
|
||||
expect(write()).toMatchObject({ ok: false, downgraded: [id], dismissed: [] });
|
||||
expect(write('closing summary: it names no defect')).toMatchObject({ ok: true, downgraded: [], dismissed: [id] });
|
||||
expect(ledgerBlockers(ledger)).toEqual([]);
|
||||
expect(verify(ledger).rows.find(r => r.kind === 'JUDGE')!.status).toBe('OK');
|
||||
fs.rmSync(d, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
test('coordinated denials are candidates', () => {
|
||||
for (const line of ['No CRITICAL or HIGH findings remain.', 'Nothing CRITICAL or HIGH remains.', 'No CRITICAL/HIGH findings.', 'No CRITICAL, HIGH or MEDIUM issues.', 'Nothing at CRITICAL severity remains.', 'No CRITICAL — the fix is sound.'])
|
||||
expect([line, summaryCandidate(line)]).toEqual([line, true]);
|
||||
});
|
||||
test('negative controls: a locus, an exception clause, a lead label, a field or a JSON finding is no candidate and keeps its label', () => {
|
||||
for (const line of ['No CRITICAL issues except the token leak in log.ts:3.', 'Nothing else is CRITICAL, but the retry loop never ends.', 'The token leak in log.ts:3 is CRITICAL.',
|
||||
'The leak in bin/gstack-log:3 is CRITICAL.', 'CRITICAL: the retry loop never ends.', 'The retry loop never ends. Severity: High', 'Not checked. Blocks: no. CRITICAL', '{"severity":"CRITICAL","summary":"x"}',
|
||||
'The retry loop in the upload handler never ends, which is CRITICAL.', 'Path traversal in the download route lets any user read arbitrary files; this is CRITICAL.',
|
||||
'The session token is written to the debug log on every request, a CRITICAL leak.', 'Nothing guards the write path the failure of which is CRITICAL',
|
||||
'There is no CRITICAL lock around the refund write, so two workers double-pay.', 'The patch closes the CRITICAL window only on Linux; on macOS the race remains.']) {
|
||||
expect([line, summaryCandidate(line)]).toEqual([line, false]);
|
||||
expect(parseFindings('fix-check', line).length).toBe(1);
|
||||
}
|
||||
const lowered = judge([{ id: 'r#1', reader: 'r', label: 'CRITICAL', raw: 'No CRITICAL issues except the token leak in log.ts:3.', output_sha256: 'x' }], [{ id: 'r#1', severity: 'INFORMATIONAL', reason: 'summary' }]);
|
||||
expect(lowered).toMatchObject({ ok: false, downgraded: ['r#1'] });
|
||||
});
|
||||
});
|
||||
|
||||
describe('the blocker count keys on Blocks (GST-1031 case 3)', () => {
|
||||
// Value: protects=a CRITICAL its reader marked Blocks: no is not counted as a blocker by the ledger count, the row count or Step
|
||||
// 9.4's classify, while Blocks: yes, an unstated Blocks and a lead promotion still block; fails_when=REVIEW_LOOP reports a
|
||||
// Blocks: no note as found or unrecorded, or a Blocks: yes CRITICAL stops blocking; why_new=ggstack#52: a lane hand-added
|
||||
// fixed rows to get a receipt; seam=ledgerBlockers, summarizeReviewLoop, isBlocking; no_claim=what the reader meant
|
||||
const OUT = '1. Drops rows in q.ts:7. Severity: Critical. Blocks: no.\n\n2. Leaks a token in log.ts:3. Severity: Critical. Blocks: yes.\n\n3. Races in a.ts:9. Severity: Critical.\n\n4. Slows down in b.ts:2. Severity: Medium. Blocks: no.\n';
|
||||
const run = (judged?: string) => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gst1031b-'));
|
||||
const ledger = path.join(dir, 'l.jsonl');
|
||||
const merged = path.join(dir, 'm.jsonl');
|
||||
fs.writeFileSync(path.join(dir, 'out.txt'), OUT);
|
||||
collect(ledger, 'outside-voice', OUT, merged, path.join(dir, 'out.txt'));
|
||||
if (judged !== undefined) {
|
||||
fs.writeFileSync(path.join(dir, 'j.jsonl'), judged);
|
||||
fs.appendFileSync(ledger, JSON.stringify({ event: 'judge', ok: true, judged: path.join(dir, 'j.jsonl'), judged_sha256: promptHash(judged) }) + '\n');
|
||||
}
|
||||
const ids = ledgerBlockers(ledger).map(b => b.id);
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
return ids;
|
||||
};
|
||||
test('ledgerBlockers skips a Blocks: no CRITICAL and keeps Blocks: yes, unstated and a lead promotion', () => {
|
||||
expect(run()).toEqual(['outside-voice#2', 'outside-voice#3']);
|
||||
const promote = ['outside-voice#1', 'outside-voice#2', 'outside-voice#3'].map(id => JSON.stringify({ id, severity: 'CRITICAL' })).join('\n') + '\n' + JSON.stringify({ id: 'outside-voice#4', severity: 'CRITICAL', reason: 'it loses rows' }) + '\n';
|
||||
expect(run(promote)).toEqual(['outside-voice#2', 'outside-voice#3', 'outside-voice#4']);
|
||||
});
|
||||
test('a verb "blocks no" is no Blocks field: the located CRITICAL still counts, on the ledger and in a native finding', () => {
|
||||
for (const out of ['CRITICAL: The lock in queue.ts:40 blocks no other writer, so two passes lose findings.\n', '1. **CRITICAL** — cache.ts:12 evicts on read; it blocks no request with a stale token.\n', '- CRITICAL src/auth.ts:9 token check is skipped; it blocks no attacker.\n']) {
|
||||
const d = fs.mkdtempSync(path.join(os.tmpdir(), 'gst1031v-'));
|
||||
const c = collect(path.join(d, 'l.jsonl'), 'outside-voice', out, path.join(d, 'm.jsonl'));
|
||||
expect([out, c.lines[0]!.stated?.blocks, ledgerBlockers(path.join(d, 'l.jsonl')).map(b => b.id)]).toEqual([out, undefined, ['outside-voice#1']]);
|
||||
fs.rmSync(d, { recursive: true, force: true });
|
||||
}
|
||||
for (const raises of [true, false]) expect(labels('adversarial-native', '**1. Race in a.ts:3. FIXABLE.**\nIt blocks no attacker.\nSeverity: Low.', raises)).toEqual(['CRITICAL']);
|
||||
const q = fs.mkdtempSync(path.join(os.tmpdir(), 'gst1031y-'));
|
||||
const y = collect(path.join(q, 'l.jsonl'), 'outside-voice', 'CRITICAL: auth.ts:9 skips authentication. The previous note said `Blocks: no`. Blocks yes.\n', path.join(q, 'm.jsonl'));
|
||||
expect([y.lines[0]!.stated?.blocks, ledgerBlockers(path.join(q, 'l.jsonl')).map(b => b.id)]).toEqual(['yes', ['outside-voice#1']]);
|
||||
fs.rmSync(q, { recursive: true, force: true });
|
||||
const d = fs.mkdtempSync(path.join(os.tmpdir(), 'gst1031j-'));
|
||||
collect(path.join(d, 'l.jsonl'), 'specialist', JSON.stringify({ severity: 'CRITICAL', path: 'lib/auth.ts', line: 9, summary: 'the old guard, which a reader marked `Blocks: no`, is gone so any caller gets admin' }) + '\n', path.join(d, 'm.jsonl'));
|
||||
expect(ledgerBlockers(path.join(d, 'l.jsonl')).map(b => b.id)).toEqual(['specialist#1']);
|
||||
fs.rmSync(d, { recursive: true, force: true });
|
||||
});
|
||||
test('classify and the row count key on Blocks, and a raw holding both Blocks: yes and no blocks', () => {
|
||||
expect(isBlocking({ severity: 'CRITICAL', stated: { severity: 'Critical', blocks: 'no' } })).toBe(false);
|
||||
expect(isBlocking({ severity: 'CRITICAL', stated: { severity: 'Critical', blocks: 'yes' } })).toBe(true);
|
||||
expect(isBlocking({ severity: 'CRITICAL' })).toBe(true);
|
||||
const row = (stated?: object) => ({ skill: 'review', via: 'ship', timestamp: '2026-11-02T00:00:00Z', findings: [{ path: 'q.ts', line: 7, severity: 'CRITICAL', ...(stated ? { stated } : {}) }] });
|
||||
expect(summarizeReviewLoop([row({ blocks: 'no' })], '2026-11-01T00:00:00Z').blocking_found).toBe(0);
|
||||
expect(summarizeReviewLoop([row()], '2026-11-01T00:00:00Z')).toMatchObject({ blocking_found: 1, unrecorded: 1 });
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gst1031c-'));
|
||||
const both = collect(path.join(dir, 'l.jsonl'), 'outside-voice', 'Leaks a token in log.ts:3. Severity: Critical. Blocks: yes. Retry is fine. Blocks: no.\n', path.join(dir, 'm.jsonl'));
|
||||
expect(both.lines.map(l => l.stated?.blocks)).toEqual(['yes']);
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
});
|
||||
});
|
||||
@@ -21,7 +21,8 @@ describe('parseFindings is never weaker than main (GST-976 invariant)', () => {
|
||||
// why_new=three GST-976 review rounds each found a shape the row tables missed; seam=parseFindings; no_claim=that main's label was
|
||||
// right (it is the floor), or coverage of stored private reader outputs (checked outside the repo). GST-987: the gate class (and every
|
||||
// risk class) keeps this rule; outside it the native voice's FIXABLE beside Blocks: no no longer counts as a raise, so such a fielded
|
||||
// finding collects at its field unless a lead, a label heading or Blocks: yes raises it
|
||||
// finding collects at its field unless a lead, a label heading or Blocks: yes raises it. GST-1031 ruling A: the gate and risk classes
|
||||
// drop that raise too, so on every class FIXABLE raises only a finding that does not state Blocks: no
|
||||
test.each([['gate', true], ['code', false]] as const)('every pinned input, reader fixture and generated field shape (%s class)', (_cls, raises) => {
|
||||
const { entries, shapes } = JSON.parse(fs.readFileSync(path.join(ROOT, 'test/fixtures/review-ledger/main-labels.json'), 'utf8'));
|
||||
const { bodies, fields: tails, joins, readers, generated } = shapes;
|
||||
@@ -37,7 +38,7 @@ describe('parseFindings is never weaker than main (GST-976 invariant)', () => {
|
||||
findings++;
|
||||
const fields = [...f.raw.matchAll(FIELD)].map(m => (m[1] ?? m[2])!);
|
||||
const single = fields.length === 1 && fields[0]!.match(LABEL)!.length === 1 ? fields[0]!.match(LABEL)![0] : undefined;
|
||||
const raised = /Blocks:?\**:?\s*\**\s*yes/i.test(f.raw) || (e.reader.startsWith('adversarial-native') && /FIXABLE/.test(f.raw) && (raises || !/Blocks:?\**:?\s*\**\s*no\b/i.test(f.raw))) || (raises ? LEAD : LEAD_NO_FIXABLE).test(f.raw) || HEADING.test(output);
|
||||
const raised = /Blocks:?\**:?\s*\**\s*yes/i.test(f.raw) || (e.reader.startsWith('adversarial-native') && /FIXABLE/.test(f.raw) && !/Blocks:?\**:?\s*\**\s*no\b/i.test(f.raw)) || (raises ? LEAD : LEAD_NO_FIXABLE).test(f.raw) || HEADING.test(output);
|
||||
const ok = single === undefined ? (rank(f.label) ?? 0) >= (rank(e.main[i]) ?? 0)
|
||||
: (rank(f.label) ?? 0) >= (rank(single) ?? 0) && (raised || (rank(f.label) ?? 0) === (rank(single) ?? 0));
|
||||
if (!ok) bad.push(`${e.reader} main=${e.main[i]} got=${f.label} field=${single ?? '-'} ${JSON.stringify(f.raw.slice(0, 100))}`);
|
||||
|
||||
+108
-5
@@ -144,18 +144,20 @@ function treeLine(dir: string): string {
|
||||
return tree.status === 0 && top.status === 0 ? `REVIEWED_TREE: ${tree.stdout.trim()} ${top.stdout.trim()}\n` : '';
|
||||
}
|
||||
const briefDir = (prompt: string) => /Workspace: the checkout under review is (.+?)\. Your shell and file tools/.exec(prompt)?.[1] ?? '';
|
||||
function host(dir: string, env: NodeJS.ProcessEnv, o: { output?: (title: string) => string; reportFrom?: (title: string) => string; fail?: (title: string) => boolean; reject?: (title: string) => boolean; createThrows?: (title: string) => 'before' | 'after' | undefined; pollsToFinish?: number; lateSummary?: boolean } = {}) {
|
||||
function host(dir: string, env: NodeJS.ProcessEnv, o: { output?: (title: string) => string; reportFrom?: (title: string) => string; fail?: (title: string) => boolean; reject?: (title: string) => boolean; createThrows?: (title: string) => 'before' | 'after' | undefined; pollsToFinish?: number; lateSummary?: boolean; hang?: (title: string) => boolean; stopped?: (title: string) => boolean; keepSummary?: boolean; epoch?: number } = {}) {
|
||||
let clock = 0;
|
||||
const subs = new Map<string, Sub>();
|
||||
const creates: Sub[] = [];
|
||||
const starts: number[][] = [];
|
||||
const stops: string[] = [];
|
||||
const reply = (s: Sub) => treeLine(o.reportFrom?.(s.title) ?? briefDir(s.prompt)) + (o.output?.(s.title) ?? cleanReply(s.title));
|
||||
const advance = () => {
|
||||
for (const s of subs.values()) {
|
||||
if (s.late && !s.summary) { s.summary = reply(s); continue; }
|
||||
if (s.status !== 'working') continue;
|
||||
if (s.status !== 'working' || o.hang?.(s.title)) continue;
|
||||
if (++s.polls < (o.pollsToFinish ?? 1)) continue;
|
||||
if (o.fail?.(s.title)) s.status = 'failed';
|
||||
if (o.stopped?.(s.title)) { s.status = 'stopped'; if (o.keepSummary) s.summary = reply(s); }
|
||||
else if (o.fail?.(s.title)) { s.status = 'failed'; if (o.keepSummary) s.summary = reply(s); }
|
||||
else if (o.lateSummary && !s.late) { s.status = 'done'; s.late = true; }
|
||||
else { s.status = 'done'; s.summary = reply(s); }
|
||||
}
|
||||
@@ -185,10 +187,11 @@ function host(dir: string, env: NodeJS.ProcessEnv, o: { output?: (title: string)
|
||||
start: async ({ indices, model }: { indices: number[]; model?: unknown }) => { starts.push(indices); for (const i of indices) { const s = subs.get(String(i))!; s.status = 'working'; if (model) s.model = model; } return { started: indices.map(index => ({ index })), rejected: [] }; },
|
||||
list: async () => { advance(); return { tasks: [...subs].map(([p, s]) => ({ path: p, title: s.title, status: s.status })), total: subs.size }; },
|
||||
get: async ({ path: p }: { path: string }) => ({ path: p, status: subs.get(p)!.status, summary: subs.get(p)!.summary }),
|
||||
stop: async ({ paths }: { paths: string[] }) => { stops.push(...paths); for (const p of paths) subs.get(p)!.status = 'stopped'; return { results: paths.map(p => ({ path: p, ok: true })) }; },
|
||||
},
|
||||
thread: { get: async () => ({ cost: { tokens: 1000 + clock, credits: 1 + clock / 1000 } }) },
|
||||
};
|
||||
return { tools, subs, creates, starts, sleep: async (ms: number) => { clock += ms; }, now: async () => 1_000_000 + clock };
|
||||
return { tools, subs, creates, starts, stops, sleep: async (ms: number) => { clock += ms; }, now: async () => (o.epoch ?? 1_000_000) + clock };
|
||||
}
|
||||
|
||||
const PROGRAM = fs.readFileSync(path.join(ROOT, 'lib/capy-review-pass.js'), 'utf8');
|
||||
@@ -362,7 +365,9 @@ describe('Capy pass program (fake host, real bins)', () => {
|
||||
expect(await runPass(h, { ...base, stage: 'voices', laneModel: undefined })).toContain("STOP: the outside voice needs input.models['outside-voice']");
|
||||
expect(h.creates.some(c => c.title.startsWith('ship x7p1 ') && /adversarial|outside/.test(c.title))).toBe(false);
|
||||
const models = { 'outside-voice': { model_id: 'openai/gpt-6.1-sol' }, 'adversarial-native': { model_id: 'anthropic/claude-opus-5-5', reasoning_mode: 'high' } };
|
||||
expect(await runPass(h, { ...base, stage: 'voices', models })).toContain('NEXT: judge the voices');
|
||||
const voices = await runPass(h, { ...base, stage: 'voices', models });
|
||||
expect(voices).toContain('NEXT: judge the voices');
|
||||
expect(voices).toContain('OUTSIDE_RECORD: "outside_provider":"host-subagent","outside_model":"openai/gpt-6.1-sol","outside_status":"completed"');
|
||||
expect(h.creates.find(c => c.title === 'ship x7p1 outside-voice')!.model).toEqual(models['outside-voice']);
|
||||
expect(h.creates.find(c => c.title === 'ship x7p1 adversarial-native')!.model).toEqual(models['adversarial-native']);
|
||||
}, 240_000);
|
||||
@@ -489,6 +494,89 @@ describe('Capy pass program (fake host, real bins)', () => {
|
||||
for (const t of Object.keys(named.models)) if (t !== 'ship x17p1 outside-voice') expect(over.models[t]).toEqual(named.models[t]);
|
||||
}, 300_000);
|
||||
|
||||
// Value: protects=GST-1104 (TRUT-7, truther#1159): an outside voice that runs past the stuck limit, or ends stopped or failed with no result, is stopped and recorded as unavailable coverage under its own runner and model, and Step 11 goes on to judge the native voice; fails_when=the stuck subagent is left running (TRUT-7 p2 burned 16.9M tokens before anyone stopped it), the pass stops with nothing recorded, or the returned record names no host-subagent runner so the review-log row falls back to the template's codex;
|
||||
// why_new=GST-1104; seam=lib/capy-review-pass.js reader poll; no_claim=which outside model Step 11 picks
|
||||
test('an outside voice that hangs or stops with no result is stopped and recorded unavailable as the host subagent, and Step 11 goes on (GST-1104)', async () => {
|
||||
for (const [pass, o] of [['x30p1', { hang: (t: string) => t.endsWith('outside-voice') }], ['x31p1', { stopped: (t: string) => t.endsWith('outside-voice') }], ['x32p1', { fail: (t: string) => t.endsWith('outside-voice') }]] as const) {
|
||||
const { dir, env } = repo();
|
||||
const out = path.join(dir, '.state/run/p1');
|
||||
const h = host(dir, env, { ...o, epoch: Date.now() });
|
||||
const base = { base: 'main', out, pass, planFlags: ['--depth-floor', 'full'] };
|
||||
expect(await runPass(h, base)).toContain('NEXT: judge (Phase 4)');
|
||||
let res = '';
|
||||
for (let i = 0; i < 6; i++) {
|
||||
res = await runPass(h, { ...base, stage: 'voices', stuckMs: 60_000 });
|
||||
if (!res.includes('NEXT: poll')) break;
|
||||
}
|
||||
expect(res).not.toContain('STOP:');
|
||||
expect(res).toContain('COLLECT_UNAVAILABLE: reader=outside-voice');
|
||||
expect(res).toContain('OUTSIDE_RECORD: "outside_provider":"host-subagent","outside_model":"openai/gpt-6.1-sol","outside_status":"unavailable"');
|
||||
expect(res).toContain('first write this voice its own adversarial-review row with "source":"host-subagent","status":"unavailable","completed":false,"converged":false and no --finish, then the native row');
|
||||
expect(res).toContain('NEXT: judge the voices');
|
||||
if ('hang' in o) {
|
||||
const token = /REVIEW_LOG_START: adversarial-review token=(\S+)/.exec(res)![1]!;
|
||||
const record = /OUTSIDE_RECORD: (\S+)/.exec(res)![1]!;
|
||||
const log = (row: string, a: string[] = []) => spawnSync(path.join(BIN, 'gstack-review-log'), [row, ...a], { cwd: dir, env, encoding: 'utf8', timeout: 60_000 });
|
||||
expect(log('{"skill":"adversarial-review","status":"unavailable","source":"host-subagent",' + record + ',"phase":"adversarial","completed":false,"converged":false}').status).toBe(0);
|
||||
expect(log('{"skill":"adversarial-review","status":"clean","source":"in-host",' + record + ',"phase":"adversarial","completed":true,"converged":true}', ['--finish', token]).status).toBe(0);
|
||||
const rows = spawnSync(path.join(BIN, 'gstack-review-read'), [], { cwd: dir, env, encoding: 'utf8', timeout: 60_000 }).stdout.split('---CONFIG---')[0]!.trim().split('\n').map(l => JSON.parse(l));
|
||||
expect(rows.map(r => [r.source, r.outside_provider, r.outside_status, r.converged])).toEqual([['host-subagent', 'host-subagent', 'unavailable', false], ['in-host', 'host-subagent', 'unavailable', true]]);
|
||||
expect(rows[1].review_freshness).toMatchObject({ status: 'CURRENT' });
|
||||
}
|
||||
const ov = [...h.subs].find(([, s]) => s.title === 'ship ' + pass + ' outside-voice')!;
|
||||
expect(ov[1].status).not.toBe('working');
|
||||
expect(h.stops).toEqual('hang' in o ? [ov[0]] : []);
|
||||
const v = bin(dir, env, ['verify', '--ledger', path.join(out, 'review-ledger.jsonl')]);
|
||||
expect(v.stdout).toMatch(/^BRIEF outside-voice UNAVAILABLE/m);
|
||||
expect(v.stdout).not.toContain('MISMATCH');
|
||||
expect(v.stdout).toContain('unavailable=1');
|
||||
}
|
||||
}, 600_000);
|
||||
|
||||
// Value: protects=GST-1104 review finding: an outside voice that wrote its report and then ended stopped or failed is collected from its summary, never replaced by unavailable coverage; fails_when=the pass records the voice unavailable without reading the summary, silently dropping its findings;
|
||||
// why_new=GST-1104 /ship review (three readers); seam=lib/capy-review-pass.js reader poll; no_claim=what the voice found
|
||||
test('an outside voice that ends stopped or failed with a report is collected, not marked unavailable (GST-1104)', async () => {
|
||||
for (const [pass, o] of [['x33p1', { stopped: (t: string) => t.endsWith('outside-voice') }], ['x34p1', { fail: (t: string) => t.endsWith('outside-voice') }]] as const) {
|
||||
const { dir, env } = repo();
|
||||
const out = path.join(dir, '.state/run/p1');
|
||||
const h = host(dir, env, { ...o, keepSummary: true, output: (t: string) => (t.endsWith('outside-voice') ? '1. **A guard fails open** - `src/a.ts:1` accepts any input. Severity: Medium. Blocks: no.\n\nRecommendation: fix the guard because it fails open' : cleanReply(t)), epoch: Date.now() });
|
||||
const base = { base: 'main', out, pass, planFlags: ['--depth-floor', 'full'] };
|
||||
expect(await runPass(h, base)).toContain('NEXT: judge (Phase 4)');
|
||||
let res = '';
|
||||
for (let i = 0; i < 6; i++) {
|
||||
res = await runPass(h, { ...base, stage: 'voices', stuckMs: 60_000 });
|
||||
if (!res.includes('NEXT: poll')) break;
|
||||
}
|
||||
expect(res).not.toContain('STOP:');
|
||||
expect(res).not.toContain('COLLECT_UNAVAILABLE: reader=outside-voice');
|
||||
expect(res).toContain('OUTSIDE_RECORD: "outside_provider":"host-subagent","outside_model":"openai/gpt-6.1-sol","outside_status":"completed"');
|
||||
expect(res).toContain('NEXT: judge the voices');
|
||||
}
|
||||
}, 600_000);
|
||||
|
||||
// Value: protects=GST-1104 review finding: an outside voice that ends stopped or failed with a summary that is not a review is recorded unavailable and Step 11 goes on; fails_when=collect's COLLECT_NOT_REVIEWED stops the pass and every rerun stops the same way;
|
||||
// why_new=GST-1104 /ship review round 2; seam=lib/capy-review-pass.js reader collect; no_claim=what the summary says
|
||||
test('an outside voice that ends stopped with a non-review summary is recorded unavailable, not a stuck pass (GST-1104)', async () => {
|
||||
const { dir, env } = repo();
|
||||
const out = path.join(dir, '.state/run/p1');
|
||||
const h = host(dir, env, { stopped: (t: string) => t.endsWith('outside-voice'), keepSummary: true, output: (t: string) => (t.endsWith('outside-voice') ? 'Stopped before I could read the diff.' : cleanReply(t)), epoch: Date.now() });
|
||||
const base = { base: 'main', out, pass: 'x35p1', planFlags: ['--depth-floor', 'full'] };
|
||||
expect(await runPass(h, base)).toContain('NEXT: judge (Phase 4)');
|
||||
let res = '';
|
||||
for (let i = 0; i < 6; i++) {
|
||||
res = await runPass(h, { ...base, stage: 'voices', stuckMs: 60_000 });
|
||||
if (!res.includes('NEXT: poll')) break;
|
||||
}
|
||||
expect(res).not.toContain('STOP:');
|
||||
expect(res).toContain('COLLECT_UNAVAILABLE: reader=outside-voice');
|
||||
expect(res).toContain('"outside_status":"unavailable"');
|
||||
expect(res).toContain('NEXT: judge the voices');
|
||||
const v = bin(dir, env, ['verify', '--ledger', path.join(out, 'review-ledger.jsonl')]);
|
||||
expect(v.stdout).toMatch(/^BRIEF outside-voice UNAVAILABLE/m);
|
||||
expect(v.stdout).not.toContain('MISMATCH');
|
||||
expect(v.stdout).not.toContain('NOT_REVIEWED');
|
||||
}, 300_000);
|
||||
|
||||
test('an audit reader marked first settles before any specialist starts; its output is returned, never merged', async () => {
|
||||
const { dir, env } = repo();
|
||||
const out = path.join(dir, '.state/run/p1');
|
||||
@@ -706,6 +794,21 @@ describe('Capy pass program (fake host, real bins)', () => {
|
||||
expect(v).toMatch(/^OUTPUT fix-check MATCH /m);
|
||||
}, 240_000);
|
||||
|
||||
// Value: protects=decision 3.5's fix-check takes a review-fix-check start token even when the parent passes the Capy skill's
|
||||
// logStart ["review"], so its row is never a pass and a review_loop_exception can follow it; fails_when=the fix-check pass logs a
|
||||
// review start; why_new=ggstack#52 (GST-1031): a scoped fix-check logged as review counted as a second pass; seam=the pass program;
|
||||
// no_claim=what the parent writes in the row
|
||||
test('a fix-check pass given logStart ["review"] takes the review-fix-check start', async () => {
|
||||
const { dir, env } = repo();
|
||||
const files = path.join(dir, '.state/fc');
|
||||
fs.mkdirSync(files, { recursive: true });
|
||||
const fc: Record<string, string> = {};
|
||||
for (const [k, v] of [['fix_list', '1. src/a.ts: guard v80\n'], ['hunks', 'diff --git a/src/a.ts b/src/a.ts\n+export const v81 = 81;\n'], ['tests', 'bun test: 3 pass, 0 fail\n']]) { fc[k] = path.join(files, k + '.txt'); fs.writeFileSync(fc[k]!, v!); }
|
||||
const res = await runPass(host(dir, env), { base: 'main', out: path.join(dir, '.state/run/fcr'), pass: 'x52fc', planFlags: ['--pass', '1', '--last', 'full', '--fixed-below'], fixCheck: fc, logStart: ['review'] });
|
||||
expect(res).toMatch(/REVIEW_LOG_START: review-fix-check token=\S+/);
|
||||
expect(res).not.toMatch(/REVIEW_LOG_START: review token=/);
|
||||
}, 240_000);
|
||||
|
||||
test('Step 11 records one start per row it logs, even when two rows share a skill (native and outside adversarial-review)', async () => {
|
||||
const { dir, env } = repo();
|
||||
const out = path.join(dir, '.state/run/p1');
|
||||
|
||||
@@ -107,8 +107,8 @@ exec ${quote(Bun.which('cat')!)} "$@"
|
||||
|
||||
const findings = (id: string) => `${id}: current findings\nNo issues found.\nRecommendation: fix ${id} because this is the current finding.\n`;
|
||||
// INV-1: the verdict-form validator prints its verdict before the completed status.
|
||||
const unavailable = (id: string) => `${findings(id)}VERDICT: unavailable\nFINDINGS: none\nREASON: execution_failed\n`;
|
||||
const completed = (id: string) => `${findings(id)}VERDICT: clean\nFINDINGS: none\nOUTSIDE_STATUS: completed provider=codex host=claude\n`;
|
||||
const unavailable = (id: string) => `${findings(id)}VERDICT: unavailable\nFINDINGS: none\nOUTSIDE_USAGE: unknown\nREASON: execution_failed\n`;
|
||||
const completed = (id: string) => `${findings(id)}VERDICT: clean\nFINDINGS: none\nOUTSIDE_USAGE: unknown\nOUTSIDE_STATUS: completed provider=codex host=claude\n`;
|
||||
|
||||
test('fresh shells retain the current stderr and clean its exact temporary file', () => {
|
||||
const f = fixture();
|
||||
|
||||
@@ -0,0 +1,194 @@
|
||||
/**
|
||||
* CI-carried lanes end to end (GSTA-258 pieces 1, 2 and 4; design fixes A1-A4): gstack-ship-preflight, gstack-ship-release
|
||||
* finish, push --early and accept, gstack-pregate's automatic --from-run and gstack-ci-carry, against a fixture repository
|
||||
* whose gh is a fake serving one Actions world, so nothing reaches GitHub.
|
||||
*/
|
||||
import { afterEach, describe, expect, setDefaultTimeout, test } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { fakeGhScript, type FakeWorld } from './helpers/fake-gh-runs';
|
||||
import { findFilesBySuffix } from './helpers/scratch-repo';
|
||||
import { lineOf, PREFLIGHT, RELEASE, shipFixture, type ShipFx } from './helpers/ship-bin-fixture';
|
||||
|
||||
setDefaultTimeout(120_000);
|
||||
const BIN = path.dirname(RELEASE);
|
||||
const fixtures: ShipFx[] = [];
|
||||
afterEach(() => { while (fixtures.length) fixtures.pop()!.cleanup(); });
|
||||
|
||||
const FREE = "printf x >> ../free-runs; printf ' 3 pass\\n 0 fail\\nRan 3 tests across 1 file. [1.00ms]\\n' >&2";
|
||||
const UNIT = "printf x >> ../unit-runs; printf ' 2 pass\\n 0 fail\\nRan 2 tests across 1 file. [1.00ms]\\n' >&2";
|
||||
const PROFILE = { workflow: 'free.yml', events: ['pull_request', 'workflow_dispatch'], jobs: ['suite'], artifact: 'tested-tree', os: 'Linux' };
|
||||
const BASE = {
|
||||
'.gstack/pregate.json': JSON.stringify({ run_profiles: { 'ci/free': PROFILE }, ship_lanes_from: { [FREE]: { profiles: ['ci/free'], inventory: "printf 'test/a.test.ts\\ntest/b.test.ts\\n'" } } }, null, 2) + '\n',
|
||||
'.github/workflows/free.yml': 'name: Free\non: [pull_request, workflow_dispatch]\njobs:\n suite:\n runs-on: ubuntu-latest\n steps:\n - run: bun test\n',
|
||||
};
|
||||
const fx = () => { const f = shipFixture(BASE); fixtures.push(f); return f; };
|
||||
const runs = (f: ShipFx, name: string) => { const p = path.join(f.root, name); return fs.existsSync(p) ? fs.readFileSync(p, 'utf8').length : 0; };
|
||||
const at = (minutesAgo: number) => new Date(Date.now() - minutesAgo * 60_000).toISOString();
|
||||
|
||||
/** One run of free.yml at HEAD (id 7 unless named), green with a tree.json naming HEAD's tree, unless overridden. */
|
||||
function world(f: ShipFx, over: { run?: Record<string, unknown>; tree?: Record<string, unknown>; others?: Array<Record<string, unknown>>; jobs?: Array<{ name: string; conclusion: string | null }> } = {}): void {
|
||||
const head = f.git('rev-parse', 'HEAD').stdout.trim();
|
||||
const tree = f.git('rev-parse', 'HEAD^{tree}').stdout.trim();
|
||||
const run = { id: 7, run_attempt: 1, status: 'completed', conclusion: 'success', event: 'workflow_dispatch', head_sha: head, path: '.github/workflows/free.yml', created_at: at(20), run_started_at: at(20), updated_at: at(5), repository: { full_name: 'fx/fx' }, head_repository: { full_name: 'fx/fx' }, ...over.run };
|
||||
const w: FakeWorld = {
|
||||
runs: [run, ...((over.others ?? []).map(o => ({ ...run, ...o })))] as FakeWorld['runs'],
|
||||
jobs: { 7: (over.jobs ?? [{ name: 'suite', conclusion: 'success' }]).map(j => ({ ...j, completed_at: at(5) })), 9: [{ name: 'suite', conclusion: 'failure', completed_at: at(1) }] },
|
||||
trees: { 7: { run_id: 7, run_attempt: 1, event: 'workflow_dispatch', sha: head, tree, files: ['test/a.test.ts', 'test/b.test.ts'], tests: 12, os: 'Linux', ...over.tree } },
|
||||
};
|
||||
const file = path.join(f.root, 'world.json');
|
||||
fs.writeFileSync(file, JSON.stringify(w));
|
||||
f.gh(fakeGhScript(file));
|
||||
}
|
||||
const preflight = (f: ShipFx, lanes = [`free=${FREE}`, `unit=${UNIT}`]) => f.run(PREFLIGHT, ['--tests-only', '--base', 'main', ...lanes.flatMap(l => ['--lane', l])]);
|
||||
const lane = (out: string, label: string) => out.split('\n').find(l => l.startsWith(`LANE: ${label} `)) ?? '';
|
||||
|
||||
describe('Step 5 lanes: CI carry and the tree cache', () => {
|
||||
// Value: protects=a lane the base maps is carried from CI at this exact tree and never runs; any other lane runs once and is reused at the same tree;
|
||||
// fails_when=the free suite reruns locally beside a green CI run, or a passed lane reruns at an unchanged tree; why_new=GSTA-258 pieces 1-2; seam=fake gh, gstack-evidence ledger
|
||||
test('the mapped lane is carried from the green run; the other lane runs once, then is reused', () => {
|
||||
const f = fx();
|
||||
world(f);
|
||||
const first = preflight(f);
|
||||
expect(first.status).toBe(0);
|
||||
expect(lane(first.stdout, 'free')).toBe('LANE: free carried run=7/1 tests_ran=12');
|
||||
expect(lane(first.stdout, 'unit')).toMatch(/^LANE: unit pass exit=0 tests_ran=2 log=\S+$/);
|
||||
expect([runs(f, 'free-runs'), runs(f, 'unit-runs')]).toEqual([0, 1]);
|
||||
const again = preflight(f);
|
||||
expect(lane(again.stdout, 'unit')).toMatch(/^LANE: unit pass exit=0 tests_ran=2 log=\S+ reused=\S+$/);
|
||||
expect([runs(f, 'free-runs'), runs(f, 'unit-runs')]).toEqual([0, 1]);
|
||||
const chk = f.run(path.join(BIN, 'gstack-evidence'), ['check', '--label', 'free', '--expect-cmd', FREE]);
|
||||
expect(chk.stdout).toContain('source=ci runs=7/1');
|
||||
});
|
||||
|
||||
// Value: protects=A2/A3 and verifyRun's refusals: a dirty tree, an uncovered inventory, a fork run and a newer red run never carry, and a label alone never maps;
|
||||
// fails_when=any of them carries; seam=lib/ci-carry.ts findCarry through runLanes
|
||||
test('a dirty tree, an uncovered inventory, a fork run, a newer red run or a changed command runs the lane here', () => {
|
||||
const f = fx();
|
||||
const cases: Array<[string, () => void, string[]?]> = [
|
||||
['dirty_tree', () => { world(f); f.write({ 'src/b.ts': 'export const b = 3;\n' }); }],
|
||||
['inventory_uncovered', () => { f.git('checkout', '--', 'src/b.ts'); world(f, { tree: { files: ['test/a.test.ts'] } }); }],
|
||||
['fork_run', () => world(f, { run: { head_repository: { full_name: 'evil/fx' } } })],
|
||||
['newer_run_vetoes', () => world(f, { others: [{ id: 9, conclusion: 'failure', created_at: at(2), run_started_at: at(2) }] })],
|
||||
];
|
||||
let expected = 0;
|
||||
for (const [reason, setup] of cases) {
|
||||
setup();
|
||||
const out = preflight(f, [`free=${FREE}`]).stdout;
|
||||
expect([reason, out]).toEqual([reason, expect.stringContaining('CI_CARRY: none label=free (')]);
|
||||
expect(out).toContain(reason);
|
||||
expect(lane(out, 'free')).toMatch(/^LANE: free pass exit=0 tests_ran=3 /);
|
||||
expected += lane(out, 'free').includes('reused=') ? 0 : 1;
|
||||
expect(runs(f, 'free-runs')).toBe(expected);
|
||||
}
|
||||
world(f);
|
||||
const other = preflight(f, [`free=${FREE} `]).stdout;
|
||||
expect(other).not.toContain('CI_CARRY');
|
||||
expect(lane(other, 'free')).not.toContain('carried');
|
||||
});
|
||||
|
||||
// Value: protects=a CI-sourced ledger row exists only for a run gstack-evidence record-ci found and verified itself (/ship pass 1 red-team#1);
|
||||
// fails_when=caller-supplied run ids, test counts or a tree make a FRESH carried row, or an unmapped command or a missing run records one; seam=fake gh
|
||||
test('record-ci verifies the run itself: forged facts, an unmapped command or no run at HEAD record nothing', () => {
|
||||
const f = fx();
|
||||
const ev = (args: string[]) => f.run(path.join(BIN, 'gstack-evidence'), args);
|
||||
const forged = ev(['record-ci', '--label', 'free', '--runs', '1/1', '--profiles', 'anything', '--tree', f.git('rev-parse', 'HEAD^{tree}').stdout.trim(), '--tests', '999', '--finished', at(1), '--', FREE]);
|
||||
expect(forged.status).toBe(2);
|
||||
world(f, { run: { head_sha: '0'.repeat(40) } });
|
||||
const noRun = ev(['record-ci', '--label', 'free', '--base', 'main', '--', FREE]);
|
||||
expect([noRun.status, noRun.stderr]).toEqual([1, expect.stringMatching(/^CARRY_NONE: no_run: /m)]);
|
||||
world(f);
|
||||
const unmapped = ev(['record-ci', '--label', 'free', '--base', 'main', '--', 'bun run test']);
|
||||
expect([unmapped.status, unmapped.stderr]).toEqual([1, expect.stringMatching(/^CARRY_NONE: not_mapped: /m)]);
|
||||
expect(ev(['check', '--label', 'free', '--expect-cmd', FREE]).stdout).not.toContain('EVIDENCE: FRESH');
|
||||
expect(JSON.parse(ev(['carried', '--json']).stdout)).toEqual([]);
|
||||
const ok = ev(['record-ci', '--label', 'free', '--base', 'main', '--', FREE]);
|
||||
expect([ok.status, ok.stderr]).toEqual([0, expect.stringContaining('tests_ran=12 log=- source=ci runs=7/1 policy=origin/main@')]);
|
||||
const row = JSON.parse(fs.readFileSync(findFilesBySuffix(path.join(f.state, 'projects'), '-evidence.jsonl')[0]!, 'utf8').trim().split('\n').at(-1)!);
|
||||
expect(row.policy).toMatch(/^origin\/main@[0-9a-f]{12}:\.gstack\/pregate\.json$/);
|
||||
});
|
||||
});
|
||||
|
||||
describe('finish, push --early and accept (A1)', () => {
|
||||
const finishArgs = (f: ShipFx, reviewed: string, post: string) => ['finish', '--base', 'main', '--reviewed-tree', reviewed, '--docs-post', post, '--lane', `free=${FREE}`];
|
||||
const snapshot = (f: ShipFx) => {
|
||||
const out = path.join(f.root, 'post.json');
|
||||
expect(f.run(path.join(BIN, 'gstack-docs-candidate'), ['snapshot', '--out', out, '--audit-id', 'a1', '--mode', 'edit', '--base', 'main', '--docs', 'README.md']).status).toBe(0);
|
||||
return out;
|
||||
};
|
||||
const unattended = { GSTACK_SESSION_KIND: 'unattended' };
|
||||
|
||||
// Value: protects=finish never runs a mapped lane locally on the stamped tree: it defers it to accept and runs only pregate's preflight stage; accept carries
|
||||
// it from the green run at the pushed head and reruns the pregate there; fails_when=the stamped tree reruns the suite, or accept trusts a run at another tree;
|
||||
// why_new=design fix A1; seam=fake gh, gstack-evidence, gstack-pregate
|
||||
test('finish defers the mapped lane; accept carries it from the green run at the pushed head', () => {
|
||||
const f = fx();
|
||||
world(f);
|
||||
const reviewed = f.run(path.join(BIN, 'gstack-wtree'), []).stdout.trim();
|
||||
expect(f.run(RELEASE, ['version', '--base', 'main', '--bump', 'micro']).status).toBe(0);
|
||||
const post = snapshot(f);
|
||||
const pregateFile = path.join(f.work, '.gstack/tmp/pregate.json');
|
||||
const before = fs.existsSync(pregateFile) ? fs.readFileSync(pregateFile, 'utf8') : null;
|
||||
const fin = f.run(RELEASE, finishArgs(f, reviewed, post));
|
||||
expect(fin.stdout).toMatch(/^EVIDENCE: DEFERRED label=free \(/m);
|
||||
expect(lineOf(fin.stdout, 'PREGATE')).toMatch(/^PREGATE: preflight (pass|warn) on the stamped tree/);
|
||||
expect(fs.existsSync(pregateFile) ? fs.readFileSync(pregateFile, 'utf8') : null).toBe(before);
|
||||
expect(fin.stdout).toContain('gstack-ship-release accept --base main --lane');
|
||||
expect(fin.status).toBe(0);
|
||||
expect(runs(f, 'free-runs')).toBe(0);
|
||||
const notPushed = f.run(RELEASE, ['accept', '--base', 'main', '--lane', `free=${FREE}`, '--wait', '0']);
|
||||
expect(notPushed.stdout).toContain('is not origin/feat: run gstack-ship-release push first');
|
||||
const push = f.run(RELEASE, ['push', '--branch', 'feat', '--early', '--base', 'main'], unattended);
|
||||
expect(lineOf(push.stdout, 'SECRETS')).toMatch(/^SECRETS: (pass|warn) \(early push\)$/);
|
||||
expect(lineOf(push.stdout, 'PUSH')).toMatch(/^PUSH: pushed /);
|
||||
world(f, { run: { status: 'in_progress', conclusion: null } });
|
||||
const running = f.run(RELEASE, ['accept', '--base', 'main', '--lane', `free=${FREE}`, '--wait', '0']);
|
||||
expect(running.status).toBe(3);
|
||||
expect(running.stdout).toContain('CI_WAIT: running ci/free run=7/1');
|
||||
world(f, { run: { conclusion: 'failure' }, jobs: [{ name: 'suite', conclusion: 'failure' }] });
|
||||
const red = f.run(RELEASE, ['accept', '--base', 'main', '--lane', `free=${FREE}`, '--wait', '0']);
|
||||
expect(red.stdout).toContain('STOP: free.yml run 7 failed (suite)');
|
||||
world(f);
|
||||
const ok = f.run(RELEASE, ['accept', '--base', 'main', '--lane', `free=${FREE}`, '--wait', '0']);
|
||||
expect(ok.stdout).toContain('CI_WAIT: green ci/free run=7/1');
|
||||
expect(lane(ok.stdout, 'free')).toBe('LANE: free carried run=7/1 tests_ran=12');
|
||||
expect(lineOf(ok.stdout, 'EVIDENCE')).toMatch(/^EVIDENCE: FRESH label=free exit=0 tests_ran=12 .* source=ci runs=7\/1$/);
|
||||
expect(ok.stdout).toContain('CI_CARRY: accepted ci-carry tree=');
|
||||
expect(ok.stdout).toContain('NEXT: Step 18');
|
||||
expect(ok.status).toBe(0);
|
||||
expect(runs(f, 'free-runs')).toBe(0);
|
||||
const carried = JSON.parse(f.run(path.join(BIN, 'gstack-evidence'), ['carried', '--json']).stdout);
|
||||
expect(carried).toEqual([{ lane: 'free', run: 7, attempt: 1, tree: f.git('rev-parse', 'HEAD^{tree}').stdout.trim() }]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('gstack-pregate tests stage and gstack-ci-carry', () => {
|
||||
// Value: protects=the tests stage carries CI at HEAD's clean tree on its own, and --no-ci-carry or a dirty tree turns it off; fails_when=pregate reruns what CI ran, or carries a dirty tree;
|
||||
// why_new=GSTA-258 piece 1; seam=fake gh; no_claim=a small machine's remote gate obligation (covered by from-run's tests)
|
||||
test('the tests stage accepts the run on its own, and --no-ci-carry or a dirty tree does not', () => {
|
||||
const f = fx();
|
||||
world(f);
|
||||
const pregate = (extra: string[] = []) => f.run(path.join(BIN, 'gstack-pregate'), ['--stage', 'tests', '--allow-repo-commands', '--base', 'main', ...extra]);
|
||||
expect(pregate().stderr).toMatch(/^CI_CARRY: accepted ci-carry tree=[0-9a-f]{12} tests=12 runs=7\/1 profiles=ci\/free /m);
|
||||
expect(pregate(['--no-ci-carry']).stderr).not.toContain('CI_CARRY');
|
||||
f.write({ 'scratch.txt': 'x\n' });
|
||||
expect(pregate().stderr).toContain('CI_CARRY: none (dirty_tree');
|
||||
});
|
||||
|
||||
// Value: protects=the CLI finds and waits on the same evidence; fails_when=find or wait accepts a red run or prints no verdict
|
||||
test('gstack-ci-carry find and wait', () => {
|
||||
const f = fx();
|
||||
world(f);
|
||||
const cli = (args: string[]) => f.run(path.join(BIN, 'gstack-ci-carry'), args);
|
||||
const found = cli(['find', '--base', 'main']);
|
||||
expect([found.status, found.stdout]).toEqual([0, expect.stringMatching(/^CI_CARRY: accepted ci-carry tree=/)]);
|
||||
const mapped = cli(['find', '--base', 'main', '--command', FREE, '--allow-repo-commands']);
|
||||
expect(mapped.status).toBe(0);
|
||||
expect(cli(['find', '--base', 'main', '--command', 'bun run test', '--allow-repo-commands']).stdout).toContain('CI_CARRY: none (not_mapped');
|
||||
expect(cli(['wait', '--base', 'main', '--timeout', '0']).stdout).toContain('CI_WAIT: green ci/free run=7/1');
|
||||
world(f, { run: { conclusion: 'failure' }, jobs: [{ name: 'suite', conclusion: 'failure' }] });
|
||||
const red = cli(['wait', '--base', 'main', '--timeout', '0']);
|
||||
expect([red.status, red.stdout.trim()]).toEqual([1, 'CI_WAIT: red ci/free run=7/1 failing=suite']);
|
||||
expect(cli(['bogus']).status).toBe(2);
|
||||
});
|
||||
});
|
||||
@@ -308,6 +308,29 @@ describe('gstack-ship-release push: branch names', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('GST-1031: a non-behavioral harness re-pin after review is release metadata', () => {
|
||||
// Value: protects=a scripts/harness-version.json edit after review that only records changed harness files as non-behavioral at the same
|
||||
// version routes as metadata, so a clean round 2 is not sent back through 5-11.5, while a version bump, a pattern change or an entry
|
||||
// without its reason stays behavior; fails_when=the re-pin stops finish, or a bump or bare re-pin routes as metadata;
|
||||
// why_new=GSTA-150 GST-1027 (icm_01M6ERWWBZ3VWXZY07F2W5ZSFB); seam=route; no_claim=that the recorded reason is true
|
||||
const pin = (version: number, entry: Record<string, unknown>, patterns = ['test/helpers/x.ts']) => JSON.stringify({ version, patterns, files: { 'test/helpers/x.ts': entry } }) + '\n';
|
||||
const finish = (after: string) => {
|
||||
const f = fx();
|
||||
f.write({ 'scripts/harness-version.json': pin(6, { blob: 'a'.repeat(40), since: 6 }) }); f.commit('pin');
|
||||
const reviewed = wtree(f);
|
||||
const post = snapshot(f);
|
||||
f.write({ 'scripts/harness-version.json': after }); f.commit('re-pin');
|
||||
return f.run(RELEASE, ['finish', '--base', 'main', '--reviewed-tree', reviewed, '--docs-post', post, '--lane', LANE]).stdout;
|
||||
};
|
||||
test('a non-behavioral re-pin is metadata; a bump, a pattern change or an entry without its reason is behavior', () => {
|
||||
const repin = finish(pin(6, { blob: 'b'.repeat(40), since: 6, nonBehavioral: 'comment-only edit: no verdict can change' }));
|
||||
expect(lineOf(repin, 'ROUTE')).toMatch(/behavior=0 docs=0 metadata=1/);
|
||||
expect(repin).not.toContain('STOP: behavior');
|
||||
for (const after of [pin(7, { blob: 'b'.repeat(40), since: 7 }), pin(6, { blob: 'b'.repeat(40), since: 6 }), pin(6, { blob: 'b'.repeat(40), since: 6, nonBehavioral: 'comment-only edit: no verdict can change' }, ['test/helpers/**'])])
|
||||
expect(finish(after)).toContain('STOP: behavior, tests or build inputs changed since review (scripts/harness-version.json)');
|
||||
});
|
||||
});
|
||||
|
||||
describe('GSTA-167 part 4a: the post-review route stays path-only', () => {
|
||||
// Value: protects=F1/F2, a data file edited after review sends the ship back through review even when no code names it,
|
||||
// because the route never applies the data rule; fails_when=a JSON edit after review takes the docs-rechecked route; seam=classifyPath
|
||||
|
||||
@@ -248,6 +248,24 @@ describe('a restamp after a lost slot (GST-895, GBRA-218, gbrain #8134)', () =>
|
||||
const e = finish(edited.f, edited.reviewed, edited.post, ['--reviewed-base', edited.reviewedBase]);
|
||||
expect([e.status, lineOf(e.stdout, 'STOP')]).toEqual([3, behaviorStop]);
|
||||
});
|
||||
|
||||
// Value: protects=with a kept VERSION claim, the restamp's move of the branch's own fork-version row routes as metadata,
|
||||
// and a base row moved the same way still stops (GST-980); fails_when=stampedFiles returns nothing at equal versions
|
||||
// and finish routes the row as behavior; seam=lib/ship-bin-common.ts stampedFiles
|
||||
test('a kept VERSION claim: the moved fork-version row is metadata; a moved base row stops (GST-980)', () => {
|
||||
const kept = (rows: [string, string][]) => {
|
||||
const f = fx({ '.gstack/ship-policy.json': policy, VERSION: '1.0.0.0\n', 'src/notice.ts': notice(['0.9.0.0', 'old'], ['1.0.0.0', 'base']) },
|
||||
{ VERSION: '1.0.1.0\n', 'src/notice.ts': notice(['0.9.0.0', 'old'], ['1.0.0.0', 'base'], ['1.0.0.0', 'mine']) });
|
||||
const reviewed = wtree(f);
|
||||
const post = snapshot(f);
|
||||
f.write({ 'src/notice.ts': notice(...rows) }); f.commit('stamp, VERSION kept');
|
||||
return finish(f, reviewed, post, ['--reviewed-base', mergeBase(f)]);
|
||||
};
|
||||
const own = kept([['0.9.0.0', 'old'], ['1.0.0.0', 'base'], ['1.0.1.0', 'mine']]);
|
||||
expect(lineOf(own.stdout, 'ROUTE')).toEqual(expect.stringMatching(/ behavior=0 docs=0 metadata=1 tests=0 /));
|
||||
const base = kept([['0.9.0.0', 'old'], ['1.0.1.0', 'base'], ['1.0.0.0', 'mine']]);
|
||||
expect([base.status, lineOf(base.stdout, 'STOP')]).toEqual([3, behaviorStop]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('adjacent base edits (GST-384, #3631, founderstar#441)', () => {
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"bytes": 1463,
|
||||
"why": "ship: CI-carry, finish/accept split and early push (GSTA-258 piece A)"
|
||||
}
|
||||
@@ -105,6 +105,22 @@ describe('tree receipt: stamps, stamp-sensitive gates and negative fixtures', ()
|
||||
expect(width.out).toMatch(/^TREE: changed \(lib\/policy\.ts\)$/m);
|
||||
});
|
||||
|
||||
// Value: protects=the fork-version bound reads the branch's own line by its position, so a base copy of the same text
|
||||
// is still floored at the old version, and when the slot keeps its VERSION only the branch's own lines may move
|
||||
// (GST-979); fails_when=stampOnlyLine matches the branch's lines by text alone or takes the old-version floor at equal
|
||||
// versions; seam=lib/stamp-only.ts stampOnlyLine
|
||||
test('the fork-version bound holds only at the branch line\'s own position, and a kept VERSION moves only the branch\'s own lines (GST-979)', () => {
|
||||
const tpl = ["since: '{version}'"];
|
||||
const row = " { since: '1.2.3.0', id: 'x' },";
|
||||
const moved = row.replace('1.2.3.0', '1.2.5.0');
|
||||
const side = { fork: '1.2.3.0', added: new Set([row]), at: new Set([10]) };
|
||||
expect(stampOnlyLine(row, moved, '1.2.4.0', '1.2.5.0', tpl, side, 10)).toBe(true);
|
||||
expect(stampOnlyLine(row, moved, '1.2.4.0', '1.2.5.0', tpl, side, 2)).toBe(false);
|
||||
const mine = { fork: '1.2.3.0', added: new Set(["mine since: '1.2.3.0'"]), at: new Set([5]) };
|
||||
expect(stampOnlyLine("base since: '1.3.0.0'", "base since: '1.2.4.0'", '1.2.4.0', '1.2.4.0', tpl, mine, 1)).toBe(false);
|
||||
expect(stampOnlyLine("mine since: '1.2.3.0'", "mine since: '1.2.4.0'", '1.2.4.0', '1.2.4.0', tpl, mine, 5)).toBe(true);
|
||||
});
|
||||
|
||||
test('negative fixture: a dependency pin equal to the old version is `changed`, not a stamp, even inside stamp_paths', () => {
|
||||
const repo = fixture();
|
||||
const g = gated(repo);
|
||||
|
||||
Reference in new issue
Block a user