mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-11 11:10:13 +01:00
Merge main 90a170c94 into frontier-models-under-test (rule 19 grant)
Conflicts: bin/gstack-eval-plan keeps both --tier-question and --seat-metered; context-budget eval-plan ceiling set to the measured 5107 (5106.9 tokens, no headroom). Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
garrytanandcapy-ai[bot] committed
commit
7388cb3259
119 files changed
+2849
-468
No files matched your search
@@ -16,7 +16,7 @@ import { EXIT } from '../lib/headless-artifacts';
|
||||
import { extractReceipt } from '../lib/ship-receipt';
|
||||
import { evaluateApprovals, parseCitations } from '../lib/approval-citations';
|
||||
import { scopeOf, type ApprovalHit } from '../lib/approval-classes';
|
||||
import { autoplan, runPhase } from './helpers/autoplan-run-fixture';
|
||||
import { autoplan, recCarrying, runPhase } from './helpers/autoplan-run-fixture';
|
||||
import { BASE_POLICY, cleanup, makeFixtureRepo, runBin, type FixtureRepo } from './helpers/restamp-fixture';
|
||||
|
||||
const repos: FixtureRepo[] = [];
|
||||
@@ -228,7 +228,7 @@ describe('gstack-ship-receipt write enforces declared classes (APPROVAL_CITATION
|
||||
expect(autoplan(['next', '--out', out, '--plan', path.join(dir, 'plan.md'), '--no-ui']).status).toBe(0);
|
||||
runPhase(out, 'ceo');
|
||||
runPhase(out, 'eng');
|
||||
if (approve) expect(autoplan(['answer', '--by', 'human', '--out', out, '--gate-rev', '1', '--reply', 'all']).stdout).toMatch(/^APPROVED: gate_rev=1/m);
|
||||
if (approve) expect(autoplan(['answer', '--by', 'human', '--out', out, '--gate-rev', '1', '--reply', recCarrying(out)]).stdout).toMatch(/^APPROVED: gate_rev=1/m);
|
||||
const sha = JSON.parse(fs.readFileSync(path.join(out, 'run.json'), 'utf8')).artifacts['plan.md'].sha256 as string;
|
||||
return { out, sha: sha.slice(0, 8) };
|
||||
};
|
||||
@@ -237,17 +237,35 @@ describe('gstack-ship-receipt write enforces declared classes (APPROVAL_CITATION
|
||||
const other = run('docs-plan', 'Rewrite the quickstart in docs/quickstart.md with a shorter first step. Tests cover the links.', true);
|
||||
const ok = write(repo, `Approved by Garry 2026-10-16 via ${good.sha}\n`, ['--approval-run', other.out, '--approval-run', good.out]);
|
||||
expect(ok.status, ok.err).toBe(EXIT.ok);
|
||||
expect(JSON.parse(ok.out).approval.classes[0]).toMatchObject({ class: 'data-repair', citation: 'verified', binding: 'paths', via: good.sha, evidence: expect.stringMatching(/^plan run-\d+ gate_rev=1 p1=a answered_by=human$/) });
|
||||
expect(JSON.parse(ok.out).approval.classes[0]).toMatchObject({ class: 'data-repair', citation: 'verified', binding: 'paths', via: good.sha, evidence: expect.stringMatching(/^plan run-\d+ gate_rev=1 p1=b answered_by=human$/) });
|
||||
const notApproved = write(repo, `Approved by Garry 2026-10-16 via ${pending.sha}\n`, ['--approval-run', pending.out]);
|
||||
expect(notApproved.err).toMatch(/holds this plan but its gate is not APPROVED \(status gate_pending, p1 unanswered\).*\(APPROVAL_CITATION_MISSING\)$/m);
|
||||
const unrelated = write(repo, `Approved by Garry 2026-10-16 via ${other.sha}\n`, ['--approval-run', other.out]);
|
||||
expect(unrelated.err).toMatch(/plan run-\d+ gate_rev=1 p1=a answered_by=human names neither/);
|
||||
expect(unrelated.err).toMatch(/plan run-\d+ gate_rev=1 p1=b answered_by=human names neither/);
|
||||
const noRun = write(repo, `Approved by Garry 2026-10-16 via ${good.sha}\n`);
|
||||
expect(noRun.err).toMatch(/plan sha [0-9a-f]{8}: pass the autoplan run with --approval-run <dir>.*must attest "for data-repair@[0-9a-f]{12}"/);
|
||||
fs.appendFileSync(path.join(good.out, 'plan.md'), '\nsrc/commands/repair/later.ts\n');
|
||||
expect(write(repo, `Approved by Garry 2026-10-16 via ${good.sha}\n`, ['--approval-run', good.out]).err).toMatch(/plan\.md does not match its manifest hash/);
|
||||
}, 120_000);
|
||||
|
||||
test('a plan gate an agent answered in its own words is not the owner\'s approval: a plan-sha citation of it is rejected', () => {
|
||||
const repo = classRepo();
|
||||
commit(repo, REPAIR);
|
||||
const dir = path.join(repo.root, 'self-approved');
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
fs.writeFileSync(path.join(dir, 'plan.md'), '# self-approved\n\n## Implementation plan\n\nAdd a data-repair command in src/commands/repair/accepted-takes.ts: explicit, preview-first, never at upgrade. Tests cover preview and apply.\n\n## Review record\n');
|
||||
const out = path.join(dir, 'run');
|
||||
expect(autoplan(['next', '--out', out, '--plan', path.join(dir, 'plan.md'), '--no-ui']).status).toBe(0);
|
||||
runPhase(out, 'ceo');
|
||||
runPhase(out, 'eng');
|
||||
const reply = recCarrying(out);
|
||||
expect(autoplan(['answer', '--by', 'agent:test', '--words-by', 'agent:test', '--owner-reply', reply, '--out', out, '--gate-rev', '1', '--reply', reply]).stdout).toMatch(/^APPROVED: gate_rev=1/m);
|
||||
const sha = (JSON.parse(fs.readFileSync(path.join(out, 'run.json'), 'utf8')).artifacts['plan.md'].sha256 as string).slice(0, 8);
|
||||
const cited = write(repo, `Approved by Garry 2026-10-16 via ${sha}\n`, ['--approval-run', out]);
|
||||
expect(cited.status).toBe(EXIT.fail);
|
||||
expect(cited.err).toMatch(/gate was answered in agent:test's own words, not the owner's.*\(APPROVAL_CITATION_MISSING\)$/m);
|
||||
}, 120_000);
|
||||
|
||||
test('an unresolvable reference counts only as an explicit for <class>@<digest> attestation, recorded unverified, and goes stale when the effects change', () => {
|
||||
const repo = classRepo();
|
||||
commit(repo, REPAIR);
|
||||
|
||||
@@ -119,13 +119,13 @@ describe('the rendered table is display only', () => {
|
||||
});
|
||||
});
|
||||
|
||||
function autoplanRun(dir: string, answers: Array<{ by: string; words?: string; at: string; p1: string }>) {
|
||||
function autoplanRun(dir: string, answers: Array<{ by: string; words?: string; at: string; p1: string; own?: boolean }>) {
|
||||
const rows = [];
|
||||
for (const a of answers) {
|
||||
for (const id of ['d1', 'p1']) rows.push({
|
||||
schema_version: 1, run: 'run-1', id: `run-1-${id}`, label: id.toUpperCase(), title: id === 'p1' ? 'Plan approval' : 'Split the CLI [run-1-ceo-native-2]',
|
||||
options: ['a', 'b'], recommended: 'a', kind: id === 'p1' ? 'approval' : 'auto', status: 'approved', gate_rev: 1,
|
||||
chosen: id === 'p1' ? a.p1 : 'a', answered_at: a.at, reply: a.words ?? 'rec', answered_by: a.by, ...(a.by === 'human' ? {} : { owner_reply: a.words }),
|
||||
chosen: id === 'p1' ? a.p1 : 'a', answered_at: a.at, reply: a.words ?? 'rec', answered_by: a.by, ...(a.by === 'human' ? {} : a.own ? { answer_words: a.words, words_by: a.by } : { owner_reply: a.words }),
|
||||
});
|
||||
}
|
||||
fs.writeFileSync(path.join(dir, 'decisions.jsonl'), rows.map(r => JSON.stringify(r)).join('\n') + '\n');
|
||||
@@ -147,6 +147,14 @@ describe('import --autoplan', () => {
|
||||
expect(bin(['import', '--autoplan', run, '--by', 'Garry'], { cwd, env: { GSTACK_STATE_ROOT: state } }).out).toMatch(/^APPROVAL_IMPORTED: n=0 skipped=2 /);
|
||||
});
|
||||
|
||||
test("each row names its own answerer: --by names only the owner's words, an agent's own words import under the agent (GEVA-2)", () => {
|
||||
const run = tmp('appr-run-');
|
||||
autoplanRun(run, [{ by: 'agent:GSTA-36', words: 'Do it', at: '2026-10-20T21:35:42.697Z', p1: 'a' }, { by: 'agent:GSTA-36', words: 'Accepted on the standing approval', at: '2026-10-21T08:00:00.000Z', p1: 'a', own: true }]);
|
||||
const rows = importAutoplan(run, 'b', 'Garry');
|
||||
if ('code' in rows) throw new Error(rows.message);
|
||||
expect(rows.map(r => [r.by, r.words])).toEqual([['Garry, relayed by agent:GSTA-36', 'Do it'], ["agent:GSTA-36, its own decision (not the owner's words)", 'Accepted on the standing approval']]);
|
||||
});
|
||||
|
||||
test('a default run id whose YYYYMMDDHHMMSS stamp passes Luhn still imports (capy-captain-flow flake)', () => {
|
||||
const run = tmp('appr-run-');
|
||||
autoplanRun(run, [{ by: 'agent:GSTA-1', words: 'rec', at: '2026-10-24T13:15:04.000Z', p1: 'a' }]);
|
||||
|
||||
@@ -13,12 +13,12 @@ import { spawnSync } from 'node:child_process';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { attemptsOf, autoplan, journal, readRun, resultOf, review, ROOT, runPhase, SMALL_PLAN, startSmallRun } from './helpers/autoplan-run-fixture';
|
||||
import { attemptsOf, autoplan, journal, readRun, recCarrying, resultOf, review, ROOT, runPhase, SMALL_PLAN, startSmallRun } from './helpers/autoplan-run-fixture';
|
||||
|
||||
const TMP = fs.mkdtempSync(path.join(os.tmpdir(), 'autoplan-answer-'));
|
||||
afterAll(() => fs.rmSync(TMP, { recursive: true, force: true }));
|
||||
|
||||
/** A gated run with a disagreement in ceo (taste item d1) and the fixture's user challenges (uc1 ceo, uc2 eng). */
|
||||
/** A gated run with a disagreement in ceo (taste item d1), the fixture's user challenges (uc1 ceo, uc2 eng) and its open Critical/High findings (d2 ceo, d3 and d4 eng). */
|
||||
function gatedRun(name: string, extra: string[] = []): string {
|
||||
const out = startSmallRun(path.join(TMP, name), extra);
|
||||
const disagree = JSON.stringify([
|
||||
@@ -60,11 +60,12 @@ function incorporate(out: string, ids: string[], edit: string) {
|
||||
}
|
||||
|
||||
describe('answer', () => {
|
||||
test('the gate lists the disagreement as an auto item and the user challenges as approval items', () => {
|
||||
test('the gate lists the disagreement and the open Critical/High findings as auto items and the user challenges as approval items', () => {
|
||||
const out = gatedRun('shape');
|
||||
const gate = JSON.parse(fs.readFileSync(path.join(out, 'gate.json'), 'utf8'));
|
||||
expect(gate.gate_rev).toBe(1);
|
||||
expect(gate.items.map((i: any) => [i.id, i.kind, i.phase ?? null])).toEqual([['d1', 'auto', 'ceo'], ['uc1', 'approval', 'ceo'], ['uc2', 'approval', 'eng'], ['p1', 'approval', null]]);
|
||||
expect(gate.items.map((i: any) => [i.id, i.kind, i.phase ?? null])).toEqual([['d1', 'auto', 'ceo'], ['uc1', 'approval', 'ceo'], ['d2', 'auto', 'ceo'], ['uc2', 'approval', 'eng'], ['d3', 'auto', 'eng'], ['d4', 'auto', 'eng'], ['p1', 'approval', null]]);
|
||||
expect(gate.items.filter((i: any) => i.finding).map((i: any) => i.id)).toEqual(['d2', 'd3', 'd4']);
|
||||
expect(gate.items[0].why).toMatch(/voices disagree: severity: native High, outside Low/);
|
||||
}, 30_000);
|
||||
|
||||
@@ -86,26 +87,27 @@ describe('answer', () => {
|
||||
expect(fs.readFileSync(path.join(out, 'run.json'), 'utf8')).toBe(before.run);
|
||||
expect(fs.readFileSync(path.join(out, 'decisions.jsonl'), 'utf8')).toBe(before.decisions);
|
||||
expect(journal(out).length).toBe(before.journal);
|
||||
// partial: only the auto item answered; approval items stay pending; nothing reviewed changes
|
||||
// partial: only the auto item answered; approval items, and finding items whose default waits on p1 (p1b applies them), stay pending
|
||||
const partial = autoplan(['answer', '--by', 'human', '--out', out, '--gate-rev', '1', '--reply', 'd1a', '--json']);
|
||||
expect(partial.status, partial.stderr).toBe(0);
|
||||
expect(JSON.parse(partial.stdout)).toMatchObject({ status: 'gate_pending', outcome: 'gate_pending', pending: ['uc1', 'uc2', 'p1'] });
|
||||
const rows = decisions(out);
|
||||
expect(rows.find((r: any) => r.label === 'D1')).toMatchObject({ status: 'approved', chosen: 'a' });
|
||||
expect(rows.filter((r: any) => r.status === 'pending').map((r: any) => r.label)).toEqual(['UC1', 'UC2', 'P1']);
|
||||
expect(rows.filter((r: any) => r.status === 'pending').map((r: any) => r.label)).toEqual(['UC1', 'D2', 'UC2', 'D3', 'D4', 'P1']);
|
||||
}, 30_000);
|
||||
|
||||
test('`all` finalizes without another paid phase: status complete, decisions approved, no phase reopened', () => {
|
||||
test('`all` carrying the open findings finalizes without another paid phase: status complete, decisions answered, no phase reopened', () => {
|
||||
const out = gatedRun('approve');
|
||||
const counter = readRun(out).attempt_counter;
|
||||
const r = autoplan(['answer', '--by', 'human', '--out', out, '--gate-rev', '1', '--reply', 'all']);
|
||||
const reply = recCarrying(out);
|
||||
const r = autoplan(['answer', '--by', 'human', '--out', out, '--gate-rev', '1', '--reply', reply]);
|
||||
expect(r.status, r.stderr).toBe(0);
|
||||
expect(r.stdout).toMatch(/^APPROVED: gate_rev=1 as-is$/m);
|
||||
expect(r.stdout).toContain('GSTACK_RESULT: skill=autoplan status=complete run=' + out);
|
||||
const run = readRun(out);
|
||||
expect(run.status).toBe('complete');
|
||||
expect(run.attempt_counter).toBe(counter);
|
||||
expect(decisions(out).every((d: any) => d.status === 'approved' && d.reply === 'all')).toBe(true);
|
||||
expect(decisions(out).every((d: any) => d.status === (/^D[234]$/.test(d.label) ? 'overridden' : 'approved') && d.reply === reply)).toBe(true);
|
||||
expect(journal(out).filter(j => j.event === 'reopened')).toEqual([]);
|
||||
const again = autoplan(['next', '--out', out]);
|
||||
expect(again.stdout).toContain('status=complete');
|
||||
@@ -114,7 +116,7 @@ describe('answer', () => {
|
||||
|
||||
test('a non-recommended choice on a ceo item is recorded without reopening; revise reopens the changed phases, archives their bound files, and the next gate is gate_rev 2', () => {
|
||||
const out = gatedRun('reopen', ['--no-carry-forward']);
|
||||
const r = autoplan(['answer', '--by', 'human', '--out', out, '--gate-rev', '1', '--reply', 'all except d1b', '--json']);
|
||||
const r = autoplan(['answer', '--by', 'human', '--out', out, '--gate-rev', '1', '--reply', recCarrying(out, 'all', ['d1b']), '--json']);
|
||||
expect(r.status, r.stderr).toBe(0);
|
||||
const runId = readRun(out).run;
|
||||
expect(JSON.parse(r.stdout)).toMatchObject({ status: 'amend_pending', outcome: 'amend_pending', reopened: [], amendments: [`${runId}-d1`] });
|
||||
@@ -145,12 +147,12 @@ describe('answer', () => {
|
||||
expect(eng.outside.stdout).toContain('status=gate_pending');
|
||||
expect(readRun(out).gate_rev).toBe(2);
|
||||
const rows = decisions(out);
|
||||
expect(rows.filter((x: any) => x.gate_rev === 1)).toHaveLength(4);
|
||||
expect(rows.filter((x: any) => x.gate_rev === 1)).toHaveLength(7);
|
||||
expect(rows.filter((x: any) => x.gate_rev === 2 && x.status === 'pending').length).toBeGreaterThan(0);
|
||||
const staleAgain = autoplan(['answer', '--by', 'human', '--out', out, '--gate-rev', '1', '--reply', 'all']);
|
||||
expect(staleAgain.status).toBe(3);
|
||||
// closing gate_rev 2 exports a run that validates: rev-2 rows get their own ids beside rev 1's
|
||||
const approve = autoplan(['answer', '--by', 'human', '--out', out, '--gate-rev', '2', '--reply', 'all']);
|
||||
const approve = autoplan(['answer', '--by', 'human', '--out', out, '--gate-rev', '2', '--reply', recCarrying(out)]);
|
||||
expect(approve.stdout).toContain('GSTACK_RESULT: skill=autoplan status=complete run=' + out);
|
||||
expect(autoplan(['export', '--out', out]).status).toBe(0);
|
||||
const ids = decisions(out).map((x: any) => x.id);
|
||||
@@ -167,7 +169,7 @@ describe('answer', () => {
|
||||
|
||||
test('an eng-only override reopens nothing at answer; revise reruns every phase that reviewed the old plan; p1d reopens every phase; p1c keeps the gate; p1e ends incomplete', () => {
|
||||
const out = gatedRun('eng-only', ['--max-rounds', '3', '--no-carry-forward']);
|
||||
const engOnly = autoplan(['answer', '--by', 'human', '--out', out, '--gate-rev', '1', '--reply', 'all except uc2b', '--json']);
|
||||
const engOnly = autoplan(['answer', '--by', 'human', '--out', out, '--gate-rev', '1', '--reply', recCarrying(out, 'all', ['uc2b']), '--json']);
|
||||
const runId = readRun(out).run;
|
||||
expect(JSON.parse(engOnly.stdout)).toMatchObject({ outcome: 'amend_pending', reopened: [], amendments: [`${runId}-uc2`] });
|
||||
expect(readRun(out).phases.ceo.closed_at).toBeTruthy();
|
||||
@@ -178,13 +180,13 @@ describe('answer', () => {
|
||||
expect(runPhase(out, 'ceo').outside.stdout).toMatch(/PHASE_CLOSED: ceo/);
|
||||
const eng = runPhase(out, 'eng');
|
||||
expect(eng.outside.stdout).toContain('status=gate_pending');
|
||||
const interrogate = autoplan(['answer', '--by', 'human', '--out', out, '--gate-rev', '2', '--reply', 'all except p1c', '--json']);
|
||||
const interrogate = autoplan(['answer', '--by', 'human', '--out', out, '--gate-rev', '2', '--reply', recCarrying(out, 'all', ['p1c']), '--json']);
|
||||
expect(JSON.parse(interrogate.stdout)).toMatchObject({ status: 'gate_pending', outcome: 'gate_pending' });
|
||||
const revise = autoplan(['answer', '--by', 'human', '--out', out, '--gate-rev', '2', '--reply', 'all except p1d', '--json']);
|
||||
const revise = autoplan(['answer', '--by', 'human', '--out', out, '--gate-rev', '2', '--reply', recCarrying(out, 'all', ['p1d']), '--json']);
|
||||
expect(JSON.parse(revise.stdout)).toMatchObject({ outcome: 'reopened', reopened: ['ceo', 'eng'] });
|
||||
runPhase(out, 'ceo');
|
||||
runPhase(out, 'eng');
|
||||
const reject = autoplan(['answer', '--by', 'human', '--out', out, '--gate-rev', '3', '--reply', 'all except p1e']);
|
||||
const reject = autoplan(['answer', '--by', 'human', '--out', out, '--gate-rev', '3', '--reply', recCarrying(out, 'all', ['p1e'])]);
|
||||
expect(reject.status).toBe(0);
|
||||
expect(reject.stdout).toContain('GSTACK_RESULT: skill=autoplan status=incomplete run=' + out);
|
||||
expect(readRun(out).deviations).toContain('plan rejected by the owner at gate_rev 3');
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
/**
|
||||
* Field list 2, W5: the autoplan run budget on the simulated Capy host. An
|
||||
* unattended Capy run reserves the default per-voice estimate so --spend-cap
|
||||
* binds, and a refused dispatch stops the run `incomplete` with a structured
|
||||
* unattended run under --spend-cap (or on Capy) reserves the default per-voice
|
||||
* estimate so the cap binds, `--estimate-usd` persists for later phases, and a refused dispatch stops the run `incomplete` with a structured
|
||||
* stop object that export, validate and resume understand; `budget` journals
|
||||
* the owner's raise. `close --deferred` keeps a dispatched voice's
|
||||
* reservation, routes an open Critical to the gate and carries open findings
|
||||
@@ -258,3 +258,37 @@ describe('confirm rounds', () => {
|
||||
expect(rows[1].settled_by).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('the cap binds on every unattended host (GBRA-223 claim 4)', () => {
|
||||
const OFF_CAPY = { GSTACK_CAPY: '' };
|
||||
test('a capped unattended run off Capy reserves the default per voice, so the cap stops it; with no cap it reserves nothing', () => {
|
||||
const dir = path.join(r.host.base, 'offcapy');
|
||||
const out = path.join(dir, 'run');
|
||||
const first = r.ap(['next', '--out', out, '--plan', writePlan(dir), '--no-ui', '--spend-cap', '12'], OFF_CAPY);
|
||||
expect(first.status, first.stderr).toBe(0);
|
||||
expect(first.stdout).toMatch(/^ESTIMATE: 5\.00 per voice reserved for each host-subagent attempt/m);
|
||||
expect(spend(out).reserved.map((x: any) => x.estimate_usd)).toEqual([DEFAULT_HOST_ESTIMATE_USD, DEFAULT_HOST_ESTIMATE_USD]);
|
||||
phaseVia(r, out, 'ceo', NONE);
|
||||
const stopped = r.ap(['next', '--out', out], OFF_CAPY);
|
||||
expect(stopped.status).toBe(3);
|
||||
expect(stopped.stderr).toMatch(/\(SPEND_CAP_EXCEEDED\)$/m);
|
||||
const freeDir = path.join(r.host.base, 'offcapy-nocap');
|
||||
const free = path.join(freeDir, 'run');
|
||||
const uncapped = r.ap(['next', '--out', free, '--plan', writePlan(freeDir), '--no-ui'], OFF_CAPY);
|
||||
expect(uncapped.status, uncapped.stderr).toBe(0);
|
||||
expect(uncapped.stdout).not.toMatch(/^ESTIMATE:/m);
|
||||
expect(spend(free).reserved.map((x: any) => x.estimate_usd)).toEqual([0, 0]);
|
||||
});
|
||||
|
||||
test('--estimate-usd persists: later phases reserve it without the flag', () => {
|
||||
const s = start(['--spend-cap', '40', '--estimate-usd', '3']);
|
||||
expect(spend(s.out).reserved.map((x: any) => x.estimate_usd)).toEqual([3, 3]);
|
||||
expect(readRunJson(s.out).estimate_usd).toBe(3);
|
||||
phaseVia(r, s.out, 'ceo', NONE);
|
||||
expect(spend(s.out).settled.map((x: any) => x.reserved_usd)).toEqual([3, 3]);
|
||||
const eng = r.ap(['next', '--out', s.out]);
|
||||
expect(eng.status, eng.stderr).toBe(0);
|
||||
expect(eng.stdout).toMatch(/^PHASE: eng /m);
|
||||
expect(spend(s.out).reserved.map((x: any) => x.estimate_usd)).toEqual([3, 3]);
|
||||
});
|
||||
});
|
||||
@@ -12,7 +12,7 @@ import { afterAll, describe, expect, test } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { RESULT_CODES } from '../lib/result-codes';
|
||||
import { ROOT, SMALL_PLAN, boundOf, resultOf } from './helpers/autoplan-run-fixture';
|
||||
import { ROOT, SMALL_PLAN, boundOf, recCarrying, resultOf } from './helpers/autoplan-run-fixture';
|
||||
import { capyRunner, phaseVia, readJsonl, readRunJson, writePlan, type CapyRunner } from './helpers/autoplan-capy';
|
||||
|
||||
const r = capyRunner();
|
||||
@@ -55,11 +55,11 @@ describe('unattended runs on the Capy host', () => {
|
||||
phaseVia(r, s.out, 'ceo');
|
||||
expect(phaseVia(r, s.out, 'eng').outside.stdout).toContain('status=gate_pending');
|
||||
expectReviewedHashes(s.out);
|
||||
expect(ok(r.ap(['answer', '--by', 'human', '--out', s.out, '--gate-rev', String(rev), '--reply', 'all except p1d'])).stdout).toMatch(/^REOPENED: ceo,eng \(Eng last\) reason=revise$/m);
|
||||
expect(ok(r.ap(['answer', '--by', 'human', '--out', s.out, '--gate-rev', String(rev), '--reply', recCarrying(s.out, 'all', ['p1d'])])).stdout).toMatch(/^REOPENED: ceo,eng \(Eng last\) reason=revise$/m);
|
||||
}
|
||||
phaseVia(r, s.out, 'ceo');
|
||||
phaseVia(r, s.out, 'eng');
|
||||
expect(ok(r.ap(['answer', '--by', 'human', '--out', s.out, '--gate-rev', '3', '--reply', 'all'])).stdout).toContain('status=complete');
|
||||
expect(ok(r.ap(['answer', '--by', 'human', '--out', s.out, '--gate-rev', '3', '--reply', recCarrying(s.out)])).stdout).toContain('status=complete');
|
||||
ok(r.ap(['export', '--out', s.out]));
|
||||
const v = ok(r.validate(s.out));
|
||||
expect(v.stdout).toMatch(/^ARTIFACT_VALID: run=\S+ status=complete/m);
|
||||
@@ -83,7 +83,7 @@ describe('unattended runs on the Capy host', () => {
|
||||
const gated = ok(r.ap(['submit', '--out', s.out, '--phase', 'eng', '--voice', 'outside', '--unavailable', 'no cross-family subagent on this host']));
|
||||
expect(gated.stdout).toMatch(/^PHASE_CLOSED: eng .*coverage=native_only/m);
|
||||
expect(gated.stdout).toContain('status=gate_pending');
|
||||
expect(ok(r.ap(['answer', '--by', 'human', '--out', s.out, '--gate-rev', '1', '--reply', 'all'])).stdout).toContain('status=complete');
|
||||
expect(ok(r.ap(['answer', '--by', 'human', '--out', s.out, '--gate-rev', '1', '--reply', recCarrying(s.out)])).stdout).toContain('status=complete');
|
||||
ok(r.ap(['export', '--out', s.out]));
|
||||
expect(ok(r.validate(s.out)).stdout).toMatch(/^ARTIFACT_VALID: run=\S+ status=complete attested=1$/m);
|
||||
expect(fs.readFileSync(path.join(s.out, 'review-record.md'), 'utf8')).toMatch(/attested/);
|
||||
@@ -94,7 +94,7 @@ describe('unattended runs on the Capy host', () => {
|
||||
const s = start();
|
||||
phaseVia(r, s.out, 'ceo');
|
||||
phaseVia(r, s.out, 'eng');
|
||||
ok(r.ap(['answer', '--by', 'human', '--out', s.out, '--gate-rev', '1', '--reply', 'all except p1d']));
|
||||
ok(r.ap(['answer', '--by', 'human', '--out', s.out, '--gate-rev', '1', '--reply', recCarrying(s.out, 'all', ['p1d'])]));
|
||||
phaseVia(r, s.out, 'ceo');
|
||||
phaseVia(r, s.out, 'eng');
|
||||
const file = path.join(s.out, 'decisions.jsonl');
|
||||
@@ -102,7 +102,7 @@ describe('unattended runs on the Capy host', () => {
|
||||
const manifest = readRunJson(s.out);
|
||||
delete manifest.gstack_version;
|
||||
fs.writeFileSync(path.join(s.out, 'run.json'), JSON.stringify(manifest, null, 2) + '\n');
|
||||
const res = ok(r.ap(['answer', '--by', 'human', '--out', s.out, '--gate-rev', '2', '--reply', 'all']));
|
||||
const res = ok(r.ap(['answer', '--by', 'human', '--out', s.out, '--gate-rev', '2', '--reply', recCarrying(s.out)]));
|
||||
expect(res.stdout).toMatch(/^GSTACK_VERSION_CHANGED: run unknown -> /m);
|
||||
expect(res.stdout).toContain('status=complete');
|
||||
ok(r.ap(['export', '--out', s.out]));
|
||||
@@ -150,7 +150,7 @@ describe('conformance: one run through every mechanism', () => {
|
||||
const accept = JSON.parse(fs.readFileSync(path.join(s.out, 'gate.json'), 'utf8')).items.find((i: any) => i.title.startsWith('Gate the backoff')).id;
|
||||
// partial, then full
|
||||
expect(ok(c.ap(['answer', '--by', 'human', '--out', s.out, '--gate-rev', '1', '--reply', `${accept}b`])).stdout).toMatch(/^GATE_PENDING: p1/m);
|
||||
const full = ok(c.ap(['answer', '--by', 'human', '--out', s.out, '--gate-rev', '1', '--reply', 'all']));
|
||||
const full = ok(c.ap(['answer', '--by', 'human', '--out', s.out, '--gate-rev', '1', '--reply', recCarrying(s.out)]));
|
||||
expect(full.stdout).toMatch(new RegExp(`^APPLY: ${run}-${accept} `, 'm'));
|
||||
const pendingNext = c.ap(['next', '--out', s.out]).stderr;
|
||||
expect(pendingNext).toMatch(/\(PLAN_AMEND_PENDING\)$/m);
|
||||
@@ -169,10 +169,10 @@ describe('conformance: one run through every mechanism', () => {
|
||||
for (const voice of ['native', 'outside']) ok(c.ap(['submit', '--out', s.out, '--phase', 'ceo', '--voice', voice, '--result', resultOf(s.out, 'ceo', voice as 'native' | 'outside'), '--model', voice === 'native' ? 'anthropic/claude-opus-5-5' : 'openai/gpt-6.1-sol']));
|
||||
phaseVia(c, s.out, 'eng');
|
||||
expect(readRunJson(s.out).gate_rev).toBe(2);
|
||||
ok(c.ap(['answer', '--by', 'human', '--out', s.out, '--gate-rev', '2', '--reply', 'all except p1d']));
|
||||
ok(c.ap(['answer', '--by', 'human', '--out', s.out, '--gate-rev', '2', '--reply', recCarrying(s.out, 'all', ['p1d'])]));
|
||||
phaseVia(c, s.out, 'ceo');
|
||||
phaseVia(c, s.out, 'eng');
|
||||
expect(ok(c.ap(['answer', '--by', 'human', '--out', s.out, '--gate-rev', '3', '--reply', 'all'])).stdout).toContain('status=complete');
|
||||
expect(ok(c.ap(['answer', '--by', 'human', '--out', s.out, '--gate-rev', '3', '--reply', recCarrying(s.out)])).stdout).toContain('status=complete');
|
||||
ok(c.ap(['export', '--out', s.out]));
|
||||
expect(ok(c.validate(s.out)).stdout).toMatch(/^ARTIFACT_VALID: run=\S+ status=complete/m);
|
||||
const final = readRunJson(s.out);
|
||||
|
||||
@@ -31,7 +31,7 @@ const decisions = (out: string) => readJsonl(path.join(out, 'decisions.jsonl'));
|
||||
const journal = (out: string) => readJsonl(path.join(out, 'attempts.jsonl'));
|
||||
|
||||
/** Gate 1 with one recommended plan change (uc1b), answered with the owner's words, then revised to take it. */
|
||||
function answeredAndRevised(args: string[] = [], words = 'Garry: Do it', ceo: { native?: unknown[]; outside?: unknown[] } = AGREE) {
|
||||
function answeredAndRevised(args: string[] = [], words = 'Garry: Do it', ceo: { native?: unknown[]; outside?: unknown[] } = AGREE, answerArgs: string[] = []) {
|
||||
const dir = path.join(r.host.base, `carry${++n}`);
|
||||
const plan = writePlan(dir);
|
||||
const out = path.join(dir, 'run');
|
||||
@@ -39,7 +39,7 @@ function answeredAndRevised(args: string[] = [], words = 'Garry: Do it', ceo: {
|
||||
phaseVia(r, out, 'ceo', ceo);
|
||||
expect(phaseVia(r, out, 'eng', NONE).outside.stdout).toContain('status=gate_pending');
|
||||
const run = readRunJson(out).run;
|
||||
const ans = r.ap(['answer', '--out', out, '--gate-rev', '1', '--reply', 'rec', '--rec-hash', recHashOf(out), '--by', 'agent:gstack-minder', '--owner-reply', words]);
|
||||
const ans = r.ap(['answer', '--out', out, '--gate-rev', '1', '--reply', 'rec', '--rec-hash', recHashOf(out), '--by', 'agent:gstack-minder', '--owner-reply', words, ...answerArgs]);
|
||||
expect(ans.status, ans.stderr).toBe(0);
|
||||
expect(ans.stdout).toMatch(/^APPLY: /m);
|
||||
const v2 = path.join(dir, 'plan-v2.md');
|
||||
@@ -50,6 +50,58 @@ function answeredAndRevised(args: string[] = [], words = 'Garry: Do it', ceo: {
|
||||
}
|
||||
const recHashOf = (out: string) => recHash(gateOf(out));
|
||||
|
||||
describe('whose words answered the gate (GEVA-2, P7)', () => {
|
||||
test("an agent's own words are recorded as its own, never as the owner's, never carried, and imported under the agent", () => {
|
||||
const s = answeredAndRevised([], 'Accepted on the standing approval', AGREE, ['--words-by', 'agent:gstack-minder']);
|
||||
const p1 = decisions(s.out).find(d => d.gate_rev === 1 && d.label === 'P1');
|
||||
expect(p1).toMatchObject({ answer_words: 'Accepted on the standing approval', words_by: 'agent:gstack-minder' });
|
||||
expect(p1.owner_reply).toBeUndefined();
|
||||
phaseVia(r, s.out, 'ceo', NONE);
|
||||
const eng = phaseVia(r, s.out, 'eng', NONE).outside;
|
||||
expect(eng.stdout).toMatch(/^CARRY: no \(gate_rev 1 was answered in agent:gstack-minder's own words, not the owner's\)$/m);
|
||||
expect(eng.stdout).toContain('status=gate_pending');
|
||||
const rows = importAutoplan(s.out, 'feature/x', 'Garry');
|
||||
if ('code' in rows) throw new Error(rows.message);
|
||||
expect(rows[0]).toMatchObject({ by: "agent:gstack-minder, its own decision (not the owner's words)", words: 'Accepted on the standing approval' });
|
||||
expect(r.ap(['export', '--out', s.out]).status).toBe(0);
|
||||
expect(r.validate(s.out).status).toBe(0);
|
||||
});
|
||||
|
||||
test('a later answer from someone else replaces the earlier answer\'s authorship and words; nothing stale survives', () => {
|
||||
const dir = path.join(r.host.base, `carry${++n}`);
|
||||
const out = path.join(dir, 'run');
|
||||
expect(r.ap(['next', '--out', out, '--plan', writePlan(dir), '--no-ui']).status).toBe(0);
|
||||
phaseVia(r, out, 'ceo', NONE);
|
||||
phaseVia(r, out, 'eng', NONE);
|
||||
const first = r.ap(['answer', '--out', out, '--gate-rev', '1', '--reply', 'p1c', '--by', 'agent:gstack-minder', '--words-by', 'agent:gstack-minder', '--owner-reply', 'p1c']);
|
||||
expect(first.status, first.stderr).toBe(0);
|
||||
const second = r.ap(['answer', '--out', out, '--gate-rev', '1', '--reply', 'p1a', '--by', 'human']);
|
||||
expect(second.status, second.stderr).toBe(0);
|
||||
expect(second.stdout).toMatch(/^APPROVED: gate_rev=1/m);
|
||||
const p1 = decisions(out).find(d => d.gate_rev === 1 && d.label === 'P1');
|
||||
expect(p1).toMatchObject({ chosen: 'a', answered_by: 'human' });
|
||||
for (const stale of ['words_by', 'answer_words', 'answer_words_parsed', 'owner_reply', 'owner_reply_parsed']) expect(p1[stale]).toBeUndefined();
|
||||
const rows = importAutoplan(out, 'feature/x', 'Garry');
|
||||
if ('code' in rows) throw new Error(rows.message);
|
||||
expect(rows[0]!.by).not.toContain('agent:gstack-minder');
|
||||
});
|
||||
|
||||
test('--words-by names the owner or the answering agent; anything else is refused', () => {
|
||||
const dir = path.join(r.host.base, `carry${++n}`);
|
||||
const out = path.join(dir, 'run');
|
||||
expect(r.ap(['next', '--out', out, '--plan', writePlan(dir), '--no-ui']).status).toBe(0);
|
||||
phaseVia(r, out, 'ceo', AGREE);
|
||||
phaseVia(r, out, 'eng', NONE);
|
||||
const base = ['answer', '--out', out, '--gate-rev', '1', '--reply', 'rec', '--rec-hash', recHashOf(out), '--owner-reply', 'ok'];
|
||||
for (const bad of [['--by', 'agent:gstack-minder', '--words-by', 'agent:someone-else'], ['--by', 'human', '--words-by', 'agent:gstack-minder'], ['--by', 'agent:gstack-minder', '--words-by', 'Garry']]) {
|
||||
const res = r.ap([...base, ...bad]);
|
||||
expect(res.status, res.stdout).not.toBe(0);
|
||||
expect(res.stderr).toContain('--words-by must be owner or the answering agent');
|
||||
}
|
||||
expect(decisions(out).every(d => d.status === 'pending')).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('carry-forward', () => {
|
||||
test('a second gate with nothing new carries the owner\'s words: CARRIED, APPROVED, the record names its source', () => {
|
||||
const s = answeredAndRevised();
|
||||
@@ -123,20 +175,36 @@ describe('carry-forward', () => {
|
||||
expect(phaseVia(r, rec.out, 'eng', PKG).outside.stdout).toMatch(/^CARRY: no \(new item \w+: Split the backoff/m);
|
||||
});
|
||||
|
||||
test('an earlier answer in item tokens, or one that left an item unanswered, is not carried', () => {
|
||||
/** Gate 1 answered item by item (every item's recommendation as tokens) by the owner, then revised to take uc1. */
|
||||
function tokensAndRevised() {
|
||||
const dir = path.join(r.host.base, `carry${++n}`);
|
||||
const out = path.join(dir, 'run');
|
||||
expect(r.ap(['next', '--out', out, '--plan', writePlan(dir), '--no-ui']).status).toBe(0);
|
||||
phaseVia(r, out, 'ceo', AGREE);
|
||||
phaseVia(r, out, 'eng', NONE);
|
||||
const g = gateOf(out);
|
||||
const tokens = g.items.map((i: any) => `${i.id}${i.recommended}`).join(' ');
|
||||
const tokens = gateOf(out).items.map((i: any) => `${i.id}${i.recommended}`).join(' ');
|
||||
expect(r.ap(['answer', '--out', out, '--gate-rev', '1', '--reply', tokens, '--by', 'human']).status).toBe(0);
|
||||
const v2 = path.join(dir, 'plan-v2.md');
|
||||
fs.writeFileSync(v2, V2);
|
||||
expect(r.ap(['revise', '--out', out, '--plan', v2, '--incorporates', 'uc1']).status).toBe(0);
|
||||
phaseVia(r, out, 'ceo', NONE);
|
||||
expect(phaseVia(r, out, 'eng', NONE).outside.stdout).toMatch(/^CARRY: no \(gate_rev 1 was answered with item tokens, not rec or "do it"\)$/m);
|
||||
return { out, tokens, p1: decisions(out).find(x => x.gate_rev === 1 && x.label === 'P1') };
|
||||
}
|
||||
|
||||
test('an item-token answer carries when the later gate asks nothing new: the owner\'s p1 choice and verbatim tokens are the basis', () => {
|
||||
const s = tokensAndRevised();
|
||||
const eng = phaseVia(r, s.out, 'eng', NONE).outside;
|
||||
expect(eng.status, eng.stderr).toBe(0);
|
||||
expect(eng.stdout).toContain(`CARRIED: gate_rev=2 <- gate_rev=1 words=${JSON.stringify(s.tokens)} by=human reason=no new choice; plan unchanged since the accepted changes`);
|
||||
expect(eng.stdout).toMatch(/^APPROVED: gate_rev=2/m);
|
||||
expect(decisions(s.out).find(x => x.gate_rev === 2 && x.label === 'P1')).toMatchObject({ chosen: s.p1.chosen, carried_gate_from: s.p1.id });
|
||||
});
|
||||
|
||||
test('an item-token answer is not carried past a new item', () => {
|
||||
const s = tokensAndRevised();
|
||||
const PKG = { native: [{ id: 'F7', severity: 'Medium', title: 'Split the backoff into its own PR', user_challenge: true, changes: 'packaging', fix: 'own PR' }], outside: [] };
|
||||
expect(phaseVia(r, s.out, 'eng', PKG).outside.stdout).toMatch(/^CARRY: no \(new item \w+: Split the backoff/m);
|
||||
expect(readRunJson(s.out).status).toBe('gate_pending');
|
||||
});
|
||||
|
||||
test('an answer that left an approval item pending is not carried', () => {
|
||||
|
||||
@@ -0,0 +1,123 @@
|
||||
import { afterAll, describe, expect, test } from 'bun:test';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { parseResult } from '../lib/headless-artifacts';
|
||||
import { CEO_REVIEW_HEADINGS, ceoOnlyEvidence, listFiles } from './helpers/autoplan-ceo-only-evidence';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const read = (rel: string) => fs.readFileSync(path.join(ROOT, rel), 'utf8');
|
||||
const handoff = (...args: string[]) => spawnSync('bash', [path.join(ROOT, 'bin/gstack-ceo-mode-handoff'), ...args], { encoding: 'utf8', timeout: 10_000 });
|
||||
|
||||
describe('/autoplan --ceo-only skill text', () => {
|
||||
const skill = read('autoplan/SKILL.md');
|
||||
const section = read('autoplan/sections/ceo-only.md');
|
||||
|
||||
test('the section index routes --ceo-only to its section before any preamble block', () => {
|
||||
expect(skill).toContain('| the invocation names `--ceo-only` (read before any preamble instruction block) | `sections/ceo-only.md` |');
|
||||
});
|
||||
|
||||
test('the section forbids every question channel and names the skipped offers', () => {
|
||||
for (const s of ['Never call AskUserQuestion (native or `mcp__*`) or\nExitPlanMode', 'never ask the user in prose', 'never wait for a reply',
|
||||
'office-hours and BENEFITS_FROM', "0E's question", 'Next Steps chaining and CEO plan\npromotion', '**Never applied:** a user challenge'])
|
||||
expect(section).toContain(s);
|
||||
});
|
||||
|
||||
test('the section runs CEO only, honors --no-outside and the state root, and ends status=reviewed', () => {
|
||||
expect(section).toContain('Run Phases 0, 0.5 and 1 only');
|
||||
expect(section).toContain("`outside_status: disabled (--no-outside)`");
|
||||
expect(section).toContain('`GSTACK_STATE_ROOT`\n(the override');
|
||||
expect(section).toContain('gstack-ceo-mode-handoff "<mode>" --ceo-only');
|
||||
expect(section).toContain('`GSTACK_RESULT: skill=autoplan status=reviewed run=<absolute $_AP>`');
|
||||
for (const h of CEO_REVIEW_HEADINGS) expect(section.replace(/\`\n\`/g, '` `').replace(/\n/g, ' ')).toContain(h.replace(/\n/g, ' '));
|
||||
});
|
||||
});
|
||||
|
||||
test('a normal /autoplan run never reads sections/ceo-only.md (the parity raise in carve-guards.ts rests on it)', () => {
|
||||
const skill = read('autoplan/SKILL.md');
|
||||
expect(skill.split('ceo-only.md').length - 1).toBe(1);
|
||||
expect(skill).toContain('| the invocation names `--ceo-only` (read before any preamble instruction block) | `sections/ceo-only.md` |');
|
||||
const others = fs.readdirSync(path.join(ROOT, 'autoplan/sections')).filter(f => f.endsWith('.md') && f !== 'ceo-only.md')
|
||||
.map(f => `autoplan/sections/${f}`);
|
||||
const runner = ['bin/gstack-autoplan', 'autoplan/DRIVER.md', ...fs.readdirSync(path.join(ROOT, 'lib')).filter(f => f.startsWith('autoplan-')).map(f => `lib/${f}`)];
|
||||
for (const rel of [...others, ...runner]) expect(read(rel), rel).not.toContain('ceo-only.md');
|
||||
});
|
||||
|
||||
describe('gstack-ceo-mode-handoff --ceo-only', () => {
|
||||
test('names the CEO-only rule, not a saved preference', () => {
|
||||
const r = handoff('SELECTIVE_EXPANSION', '--ceo-only');
|
||||
expect(r.status).toBe(0);
|
||||
expect(r.stdout.trim()).toBe("Auto-selected review mode → SELECTIVE EXPANSION (CEO-only run: Step 0E's rule). Approved decisions: none.");
|
||||
});
|
||||
test('refuses --auto with --ceo-only', () => {
|
||||
const r = handoff('HOLD SCOPE', '--auto', '--ceo-only');
|
||||
expect(r.status).toBe(2);
|
||||
expect(r.stderr).toContain('--auto and --ceo-only conflict');
|
||||
});
|
||||
});
|
||||
|
||||
test('GSTACK_RESULT parses status=reviewed', () => {
|
||||
expect(parseResult('x\nGSTACK_RESULT: skill=autoplan status=reviewed run=/r')).toEqual({ skill: 'autoplan', status: 'reviewed', run: '/r' });
|
||||
});
|
||||
|
||||
describe('ceo-only evidence: a passing run and each negative control', () => {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ceo-only-evidence-'));
|
||||
afterAll(() => fs.rmSync(root, { recursive: true, force: true }));
|
||||
const runDir = path.join(root, 'run');
|
||||
fs.mkdirSync(runDir);
|
||||
const review = ['# CEO-only review: Pricing tier', '', 'Mode: HOLD SCOPE (ceo-only). Not approved.', '',
|
||||
'## User challenges (direction kept)', 'None.', '## Scope added', 'None.', '## Conservative choices', 'None.',
|
||||
'## Findings', '- High: the premise was never checked with developers; interview them first.',
|
||||
'## Decision Audit Trail', '| # | Phase | Decision |', '|---|---|---|', '| 1 | CEO | Mode HOLD SCOPE |', ''].join('\n');
|
||||
fs.writeFileSync(path.join(runDir, 'ceo-review.md'), review);
|
||||
const assistant = (...content: any[]) => ({ type: 'assistant', message: { content } });
|
||||
const user = (...content: any[]) => ({ type: 'user', message: { content } });
|
||||
const good = [
|
||||
assistant({ type: 'tool_use', id: 't1', name: 'Bash', input: { command: '~/.claude/skills/gstack/bin/gstack-ceo-mode-handoff "HOLD SCOPE" --ceo-only --decisions "none"' } }),
|
||||
user({ type: 'tool_result', tool_use_id: 't1', content: "Auto-selected review mode → HOLD SCOPE (CEO-only run: Step 0E's rule). Approved decisions: none." }),
|
||||
assistant({ type: 'tool_use', id: 't2', name: 'Agent', input: { prompt: 'native CEO review' } }),
|
||||
assistant({ type: 'text', text: 'Phases 2, 2.5, 3 and 4 skipped (CEO-only run).' }),
|
||||
];
|
||||
const output = `Review done.\nGSTACK_RESULT: skill=autoplan status=reviewed run=${runDir}`;
|
||||
const base = { transcript: good, output, strayWrites: [] as string[], mustMention: [/developer/i, /interview|talk|validat/i] };
|
||||
|
||||
test('a complete run passes', () => {
|
||||
const e = ceoOnlyEvidence(base);
|
||||
expect(e.reasons).toEqual([]);
|
||||
expect(e.runDir).toBe(runDir);
|
||||
});
|
||||
const fails = (name: string, patch: Partial<typeof base>, reason: RegExp) => test(name, () => {
|
||||
const e = ceoOnlyEvidence({ ...base, ...patch });
|
||||
expect(e.passed).toBe(false);
|
||||
expect(e.reasons.join('\n')).toMatch(reason);
|
||||
});
|
||||
fails('an AskUserQuestion call fails', { transcript: [...good, assistant({ type: 'tool_use', id: 'q', name: 'AskUserQuestion', input: {} })] }, /asked 1 question/);
|
||||
fails('an MCP AskUserQuestion variant fails', { transcript: [...good, assistant({ type: 'tool_use', id: 'q', name: 'mcp__conductor__AskUserQuestion', input: {} })] }, /asked 1 question/);
|
||||
fails('ExitPlanMode fails', { transcript: [...good, assistant({ type: 'tool_use', id: 'q', name: 'ExitPlanMode', input: {} })] }, /tool ExitPlanMode/);
|
||||
fails('a prose question fails', { output: `Which mode would you like me to use?\n${output}` }, /prose: Which mode would you like/);
|
||||
fails('a wrong terminal status fails', { output: output.replace('status=reviewed', 'status=gate_pending') }, /status=gate_pending, expected reviewed/);
|
||||
fails('a missing terminal line fails', { output: 'Review done.' }, /no GSTACK_RESULT line/);
|
||||
fails('headings without a review fail', { transcript: [assistant({ type: 'text', text: 'Done.' })] }, /no gstack-ceo-mode-handoff[\s\S]*no CEO reviewer subagent/);
|
||||
fails('loading a skipped phase fails', { transcript: [...good, assistant({ type: 'tool_use', id: 'r', name: 'Read', input: { file_path: '/h/.claude/skills/gstack/autoplan/sections/eng-phase.md' } })] }, /loaded a skipped phase/);
|
||||
fails('loading a skipped phase through Bash fails', { transcript: [...good, assistant({ type: 'tool_use', id: 'b', name: 'Bash', input: { command: 'cat /h/.claude/skills/gstack/autoplan/sections/dx-phase.md' } })] }, /loaded a skipped phase: cat /);
|
||||
fails('a write outside its roots fails', { strayWrites: ['.config/x.json'] }, /wrote 1 file\(s\) outside its roots, under \.config\/x\.json/);
|
||||
fails('a review that misses the seeded flaw fails', { mustMention: [/churn/i] }, /never names the seeded flaw/);
|
||||
test('a missing ceo-review.md fails', () => {
|
||||
const empty = path.join(root, 'empty');
|
||||
fs.mkdirSync(empty);
|
||||
const e = ceoOnlyEvidence({ ...base, output: `GSTACK_RESULT: skill=autoplan status=reviewed run=${empty}` });
|
||||
expect(e.reasons.join('\n')).toContain('ceo-review.md is missing');
|
||||
});
|
||||
test('listFiles skips the declared roots', () => {
|
||||
const home = path.join(root, 'home');
|
||||
fs.mkdirSync(path.join(home, 'state'), { recursive: true });
|
||||
fs.writeFileSync(path.join(home, 'state', 'a'), '');
|
||||
fs.writeFileSync(path.join(home, 'b'), '');
|
||||
fs.mkdirSync(path.join(home, '.bun', 'install', 'cache'), { recursive: true });
|
||||
fs.writeFileSync(path.join(home, '.bun', 'install', 'cache', 'c'), '');
|
||||
fs.mkdirSync(path.join(home, '.bun', 'bin'), { recursive: true });
|
||||
fs.writeFileSync(path.join(home, '.bun', 'bin', 'd'), '');
|
||||
expect(listFiles(home, ['state', '.bun/install/cache'])).toEqual(['.bun/bin/d', 'b']);
|
||||
});
|
||||
});
|
||||
@@ -188,7 +188,7 @@ describe('E16 end to end: every gate-producing command prints the cost, and the
|
||||
|
||||
|
||||
test('the closing submit prints GATE_SPEND/GATE_REC_COST (text) or gate_cost (--json); the owner brief carries the Cost so far sentence and render --captain a ### Cost section above the relaying trailer', () => {
|
||||
const out = gateRun('plain', {}, { challenge: false, json: true });
|
||||
const out = gateRun('plain', { severity: 'Medium' }, { challenge: false, json: true });
|
||||
const brief = spawnSync(process.execPath, [path.join(ROOT, 'bin', 'gstack-owner-brief'), 'render', '--out', out], { cwd: ROOT, env: env(), encoding: 'utf8', timeout: 30_000 });
|
||||
expect(brief.stdout).toMatch(/Cost so far: spent \d+\.\d\d, worst case \d+\.\d\d \(/);
|
||||
const captain = spawnSync(process.execPath, [path.join(ROOT, 'bin', 'gstack-owner-brief'), 'render', '--captain', '--out', out, '--allow-incomplete'], { cwd: ROOT, env: env(), encoding: 'utf8', timeout: 30_000 });
|
||||
@@ -203,7 +203,7 @@ describe('E16 end to end: every gate-producing command prints the cost, and the
|
||||
});
|
||||
|
||||
test('run at a pending gate prints the gate lines once: text mode carries GATE_SPEND/GATE_REC_COST, --json exactly one payload with gate_cost', () => {
|
||||
const out = gateRun('run-gate', {}, { challenge: false });
|
||||
const out = gateRun('run-gate', { severity: 'Medium' }, { challenge: false });
|
||||
const models = ['--native-model', 'anthropic/claude-opus-4-7', '--outside-model', 'openai/gpt-6-astra'];
|
||||
const text = autoplan(['run', '--out', out, ...models]);
|
||||
expect(text.status).toBe(0);
|
||||
|
||||
@@ -17,8 +17,9 @@ import { afterAll, describe, expect, test } from 'bun:test';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { ROOT, SMALL_PLAN, boundOf, resultOf } from './helpers/autoplan-run-fixture';
|
||||
import { ROOT, SMALL_PLAN, boundOf, recCarrying, resultOf } from './helpers/autoplan-run-fixture';
|
||||
import { capyRunner, phaseVia, readJsonl, readRunJson, writePlan } from './helpers/autoplan-capy';
|
||||
import { settledIds } from '../lib/autoplan-confirm';
|
||||
|
||||
const r = capyRunner();
|
||||
afterAll(() => r.cleanup());
|
||||
@@ -119,7 +120,7 @@ describe('accepted changes are amendments', () => {
|
||||
expect(held.status, held.stderr).toBe(0);
|
||||
expect(held.stdout).toMatch(/^GATE_PENDING: uc2, p1 unanswered \(reply with uc2a\|uc2b p1a\|p1b\|p1c\|p1d\|p1e or rec\)$/m);
|
||||
expect(held.stdout).not.toMatch(/^APPLY:/m);
|
||||
const res = answer(g.out, 1, 'rec');
|
||||
const res = answer(g.out, 1, recCarrying(g.out, 'rec'));
|
||||
expect(res.status, res.stderr).toBe(0);
|
||||
expect(res.stdout).toMatch(new RegExp(`^APPLY: ${esc(g.run)}-uc1 Ship the backoff behind a flag first .* -> accept the change both reviewers recommend.*; edit the plan, then revise$`, 'm'));
|
||||
expect(res.stdout).not.toMatch(/^REOPENED:/m);
|
||||
@@ -165,7 +166,7 @@ describe('accepted changes are amendments', () => {
|
||||
expect(asIs.stdout).not.toMatch(/^APPLY:/m);
|
||||
expect(decisions(g.out).find(d => d.label === 'UC1')).toMatchObject({ chosen: 'a', status: 'approved' });
|
||||
const w = gated(AGREE, {});
|
||||
expect(answer(w.out, 1, 'rec except uc1b').stdout).toMatch(/^APPLY:/m);
|
||||
expect(answer(w.out, 1, recCarrying(w.out, 'rec', ['uc1b'])).stdout).toMatch(/^APPLY:/m);
|
||||
const withdraw = r.ap(['revise', '--out', w.out, '--withdraw', `${w.run}-uc1`]);
|
||||
expect(withdraw.status, withdraw.stderr).toBe(0);
|
||||
expect(withdraw.stdout).toMatch(/^APPROVED: gate_rev=1 \(every accepted change is in the plan or settled\)$/m);
|
||||
@@ -179,7 +180,7 @@ describe('record-only changes settle without a plan edit', () => {
|
||||
test('a packaging change: APPLY says settle, revise --settle reopens nothing and completes, export and validate pass; a scope change never settles', () => {
|
||||
const g = gated(PACKAGING, {});
|
||||
expect(gate(g.out).items.find((i: any) => i.id === 'uc1')).toMatchObject({ recommended: 'b', changes: 'packaging' });
|
||||
const res = answer(g.out, 1, 'rec');
|
||||
const res = answer(g.out, 1, recCarrying(g.out, 'rec'));
|
||||
expect(res.status, res.stderr).toBe(0);
|
||||
expect(res.stdout).toMatch(new RegExp(`^APPLY: ${esc(g.run)}-uc1 .*; no plan edit needed: revise --settle uc1$`, 'm'));
|
||||
expect(readRunJson(g.out).status).toBe('amend_pending');
|
||||
@@ -196,7 +197,7 @@ describe('record-only changes settle without a plan edit', () => {
|
||||
expect(r.ap(['export', '--out', g.out]).status).toBe(0);
|
||||
expect(r.validate(g.out).status).toBe(0);
|
||||
const scope = gated(AGREE, {});
|
||||
expect(answer(scope.out, 1, 'rec except uc1b').stdout).toMatch(/; edit the plan, then revise$/m);
|
||||
expect(answer(scope.out, 1, recCarrying(scope.out, 'rec', ['uc1b'])).stdout).toMatch(/; edit the plan, then revise$/m);
|
||||
const refused = r.ap(['revise', '--out', scope.out, '--settle', 'uc1']);
|
||||
expect(refused.status).toBe(3);
|
||||
expect(refused.stderr).toMatch(/changes scope; only packaging or ordering changes settle.*\(AMENDMENT_NOT_SETTLEABLE\)$/m);
|
||||
@@ -229,7 +230,7 @@ describe('mixed amendments: a scope change and a packaging change', () => {
|
||||
expect(readRunJson(g.out).status).toBe('running');
|
||||
phaseVia(r, g.out, 'ceo');
|
||||
expect(phaseVia(r, g.out, 'eng').outside.stdout).toContain('status=gate_pending');
|
||||
expect(answer(g.out, 2, 'rec').stdout).toContain('status=complete');
|
||||
expect(answer(g.out, 2, recCarrying(g.out, 'rec')).stdout).toContain('status=complete');
|
||||
expect(r.validate(g.out).status).toBe(0);
|
||||
});
|
||||
test('settle and incorporate in one revise; settling alone on an unreviewed plan edit is refused', () => {
|
||||
@@ -262,7 +263,7 @@ describe('partial replies resolve against persisted decisions', () => {
|
||||
|
||||
test('validate rejects a complete run with a pending approval', () => {
|
||||
const g = gated({}, {});
|
||||
expect(answer(g.out, 1, 'all').stdout).toContain('status=complete');
|
||||
expect(answer(g.out, 1, recCarrying(g.out)).stdout).toContain('status=complete');
|
||||
const file = path.join(g.out, 'decisions.jsonl');
|
||||
fs.writeFileSync(file, decisions(g.out).map(d => JSON.stringify(d.label === 'UC2' ? { ...d, status: 'pending', chosen: null } : d)).join('\n') + '\n');
|
||||
r.ap(['export', '--out', g.out]);
|
||||
@@ -298,3 +299,100 @@ describe('reruns', () => {
|
||||
expect(decisions(g.out).find(d => d.gate_rev === 2 && d.label === demoted.id.toUpperCase())).toMatchObject({ chosen: 'b', source: 'agent' });
|
||||
});
|
||||
});
|
||||
|
||||
describe('open Critical and High findings reach the gate (GBRA-223 claim 3)', () => {
|
||||
const NONE = { native: [], outside: [] };
|
||||
const KILL = { native: [{ id: 'F1', severity: 'Critical', title: 'No kill switch for the resolver rollout', fix: 'add a config kill switch' }], outside: [] };
|
||||
test('each open Critical/High finding is an apply item that rec takes; a twin pair is one item; Medium stays a task', () => {
|
||||
const g = gated({
|
||||
native: [{ id: 'F1', severity: 'Critical', title: 'No kill switch for the resolver rollout', fix: 'add a config kill switch' }, { id: 'F2', severity: 'Medium', title: 'Name the flag consistently' }],
|
||||
outside: [{ id: 'O1', severity: 'High', title: 'No kill switch for the resolver rollout', fix: 'ship behind a kill switch' }],
|
||||
}, { native: [{ id: 'F3', severity: 'High', title: 'No test covers a hung resolver', file: 'test/resolver.test.ts', fix: 'add a hung-resolver test' }], outside: [] });
|
||||
const items = gate(g.out).items;
|
||||
const apply = items.filter((i: any) => i.finding);
|
||||
expect(apply.map((i: any) => i.title)).toEqual([
|
||||
expect.stringMatching(/^Critical: No kill switch for the resolver rollout \[.*-ceo-native-1 \+ .*-ceo-outside-1\]$/),
|
||||
expect.stringMatching(/^High: No test covers a hung resolver \[.*-eng-native-1\]$/),
|
||||
]);
|
||||
for (const i of apply) expect(i).toMatchObject({ kind: 'auto', recommended: 'a', options: [{ key: 'a', changes_plan: true }, { key: 'b', changes_plan: false }] });
|
||||
expect(apply[0].options[0].text).toContain('add a config kill switch');
|
||||
expect(apply[0].options[0].text).toContain('ship behind a kill switch');
|
||||
expect(items.find((i: any) => i.id === 'p1')).toMatchObject({ recommended: 'b' });
|
||||
expect(items.some((i: any) => /Name the flag/.test(i.title))).toBe(false);
|
||||
const rec = answer(g.out, 1, 'rec');
|
||||
expect(rec.stdout.match(/^APPLY: /gm)?.length).toBe(2);
|
||||
expect(readRunJson(g.out).status).toBe('amend_pending');
|
||||
});
|
||||
|
||||
test('a user challenge one line away does not hide an unrelated Critical: a location match alone is not a twin', () => {
|
||||
const g = gated({
|
||||
native: [{ id: 'F1', severity: 'Medium', title: 'Package the worker as its own service', file: 'plan.md', line: 10, user_challenge: true, changes: 'scope', fix: 'split the worker out' }],
|
||||
outside: [{ id: 'O1', severity: 'Critical', title: 'Webhook payload reaches the SQL query unescaped', file: 'plan.md', line: 11, fix: 'use parameterized queries' }],
|
||||
}, NONE);
|
||||
const apply = gate(g.out).items.filter((i: any) => i.finding);
|
||||
expect(apply.map((i: any) => i.title)).toEqual([expect.stringMatching(/^Critical: Webhook payload reaches the SQL query unescaped \[.*-ceo-outside-1\]$/)]);
|
||||
});
|
||||
|
||||
test('b, as-is, or a token reply that leaves the item unanswered carries the finding into the build with no plan change', () => {
|
||||
for (const pick of [(id: string) => `${id}b p1a`, () => 'as-is', () => 'p1a']) {
|
||||
const g = gated(KILL, NONE);
|
||||
const item = gate(g.out).items.find((i: any) => i.finding);
|
||||
const done = answer(g.out, 1, pick(item.id), true);
|
||||
expect(JSON.parse(done.stdout)).toMatchObject({ outcome: 'complete', reopened: [], amendments: [] });
|
||||
expect(decisions(g.out).find(d => d.label === item.id.toUpperCase())).toMatchObject({ chosen: 'b' });
|
||||
expect(readJsonl(path.join(g.out, 'tasks.jsonl')).some(t => /No kill switch/.test(t.title))).toBe(true);
|
||||
}
|
||||
const g = gated(KILL, NONE);
|
||||
const item = gate(g.out).items.find((i: any) => i.finding);
|
||||
answer(g.out, 1, 'p1a');
|
||||
expect(readJsonl(path.join(g.out, 'attempts.jsonl')).find(e => e.event === 'answered')).toMatchObject({ answer_class: 'overrides', overrides: [`${item.id}b`] });
|
||||
expect(settledIds({ out: g.out } as any).has(item.finding[0])).toBe(false);
|
||||
});
|
||||
|
||||
test('p1b applies the finding fixes its option text lists, and an applied finding is settled', () => {
|
||||
const g = gated(KILL, NONE);
|
||||
const item = gate(g.out).items.find((i: any) => i.finding);
|
||||
expect(gate(g.out).items.find((i: any) => i.id === 'p1').options[1].text).toContain(`${item.id}a`);
|
||||
expect(answer(g.out, 1, 'p1b').stdout).toMatch(/^APPLY: /m);
|
||||
expect(decisions(g.out).find(d => d.label === item.id.toUpperCase())).toMatchObject({ chosen: 'a' });
|
||||
expect(readJsonl(path.join(g.out, 'attempts.jsonl')).find(e => e.event === 'answered')).toMatchObject({ answer_class: 'plan-changes', changes: [`${item.id}a`] });
|
||||
expect(readRunJson(g.out).status).toBe('amend_pending');
|
||||
expect(settledIds({ out: g.out } as any).has(item.finding[0])).toBe(true);
|
||||
expect(settledIds({ out: g.out } as any, { findings: false }).has(item.finding[0])).toBe(false);
|
||||
});
|
||||
|
||||
test('after recommend changes an item, p1b (which takes the recommendations) needs the --rec-hash the owner saw', () => {
|
||||
const g = gated({ native: [{ id: 'F1', severity: 'High', title: 'Ship the backoff behind a flag first', user_challenge: true, changes: 'scope', fix: 'gate it behind RETRY_BACKOFF=1' }], outside: [] }, NONE);
|
||||
const item = gate(g.out).items.find((i: any) => i.id !== 'p1' && i.kind === 'auto' && i.options.some((o: any) => o.key === 'b'));
|
||||
expect(r.ap(['recommend', '--out', g.out, '--gate-rev', '1', '--item', item.id, '--option', 'b', '--basis', 'owner asked for the flag', '--by', 'agent:test']).status).toBe(0);
|
||||
const stale = answer(g.out, 1, 'p1b');
|
||||
expect(stale.status).toBe(3);
|
||||
expect(stale.stderr).toContain('GATE_REC_STALE');
|
||||
expect(decisions(g.out).every(d => d.status === 'pending')).toBe(true);
|
||||
});
|
||||
|
||||
test('a partial reply records no auto default, so a later p1b still applies the finding fix', () => {
|
||||
const g = gated({ native: [...AGREE.native, ...KILL.native.map(f => ({ ...f, id: 'F9' }))], outside: AGREE.outside }, NONE);
|
||||
const item = gate(g.out).items.find((i: any) => i.finding);
|
||||
const uc = gate(g.out).items.find((i: any) => i.id.startsWith('uc'));
|
||||
expect(answer(g.out, 1, `${uc.id}a`).stdout).toMatch(/GATE_PENDING: p1/);
|
||||
expect(decisions(g.out).find(d => d.label === item.id.toUpperCase())?.chosen ?? null).toBeNull();
|
||||
answer(g.out, 1, 'p1b');
|
||||
expect(decisions(g.out).find(d => d.label === item.id.toUpperCase())).toMatchObject({ chosen: 'a' });
|
||||
});
|
||||
|
||||
test('a same-file association never merges an unrelated Critical into a user challenge', () => {
|
||||
const g = gated({ native: [{ id: 'F1', severity: 'Medium', title: 'Split the backoff into its own PR', file: 'lib/worker.ts', user_challenge: true, changes: 'packaging', fix: 'land it separately' }],
|
||||
outside: [{ id: 'O1', severity: 'Critical', title: 'SQL injection in the lookup query', file: 'lib/worker.ts', fix: 'parameterize the query' }] }, NONE);
|
||||
expect(gate(g.out).items.some((i: any) => i.finding && /^Critical: SQL injection/.test(i.title))).toBe(true);
|
||||
});
|
||||
|
||||
test('at the round cap, an outside Critical paired by file with an unrelated native Medium still counts as open', () => {
|
||||
const dir = path.join(r.host.base, `gj${++n}`);
|
||||
const out = path.join(dir, 'run');
|
||||
expect(r.ap(['next', '--out', out, '--plan', writePlan(dir), '--no-ui', '--max-rounds', '1']).status).toBe(0);
|
||||
phaseVia(r, out, 'ceo', { native: [{ id: 'F1', severity: 'Medium', title: 'rename the config key', file: 'src/resolver.ts' }], outside: [{ id: 'O1', severity: 'Critical', title: 'SQL injection in lookup query', file: 'src/resolver.ts' }] });
|
||||
const eng = phaseVia(r, out, 'eng', NONE).outside.stdout;
|
||||
expect(eng.split('\n').filter(l => l.startsWith('ESCALATE:'))).toEqual(['ESCALATE: evidence sufficient, decision needed (ceo: 1 High open)']);
|
||||
});
|
||||
});
|
||||
@@ -7,7 +7,7 @@
|
||||
*/
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import fixture from './fixtures/autoplan-pairing/run-20261014-pairs.json';
|
||||
import { reconcile } from '../lib/autoplan-reconcile';
|
||||
import { reconcile, titleScore } from '../lib/autoplan-reconcile';
|
||||
import { challengeDefault } from '../lib/autoplan-gate';
|
||||
|
||||
const pair = (name: string) => fixture.pairs.find(p => p.name.startsWith(name))! as any;
|
||||
@@ -61,3 +61,60 @@ describe('pairing negative control from a real run', () => {
|
||||
expect(challengeDefault(ok!, challenge(same.native, 'scope')[0], challenge(same.outside, 'scope')[0])).toMatchObject({ recommended: 'b', basis: 'both voices; same remedy' });
|
||||
});
|
||||
});
|
||||
|
||||
describe('plan reviews name a file without a line (GBRA-223 claim 1)', () => {
|
||||
const row = (voice: 'native' | 'outside', n: number, f: Record<string, unknown>) => ({ schema_version: 1, run: 'r', id: `r-dx-${voice}-${n}`, phase: 'dx', voice, disposition: 'open', ...f }) as any;
|
||||
const native = row('native', 1, { severity: 'Critical', title: 'per-turn resolver call has no hard timeout or fail-open', file: 'src/resolver.ts', resolution: 'add a 2s timeout and fail open' });
|
||||
|
||||
test('a CAP-coded outside title on the same file is associated, unverified, never new', () => {
|
||||
const outside = row('outside', 1, { severity: 'Critical', title: 'CAP4:src/resolver.ts:recovery', file: 'src/resolver.ts', resolution: 'bound the call' });
|
||||
const { rows, native_only } = reconcile([native], [outside]);
|
||||
expect(rows[0]).toMatchObject({ disposition: 'unverified', basis: 'file', native_id: native.id });
|
||||
expect(rows[0]!.resolution).toStartWith('same file, agreement unverified (src/resolver.ts)');
|
||||
expect(native_only).toEqual([]);
|
||||
});
|
||||
|
||||
test('differently worded titles on the same file associate; the best title among same-file natives wins', () => {
|
||||
const other = row('native', 2, { severity: 'Medium', title: 'resolver output is not cached between turns', file: 'src/resolver.ts' });
|
||||
const outside = row('outside', 1, { severity: 'Critical', title: 'Resolver call has no timeout and can hang the turn', file: 'src/resolver.ts' });
|
||||
expect(reconcile([other, native], [outside]).rows[0]).toMatchObject({ disposition: 'unverified', basis: 'file', native_id: native.id });
|
||||
});
|
||||
|
||||
test('a CAP code is not part of the title comparison', () => {
|
||||
const outside = row('outside', 1, { severity: 'Critical', title: 'CAP4:src/resolver.ts:recovery per-turn resolver call has no hard timeout or fail-open', file: 'src/resolver.ts', resolution: 'add a 2s timeout and fail open' });
|
||||
expect(reconcile([native], [outside]).rows[0]).toMatchObject({ disposition: 'confirmed', basis: 'title' });
|
||||
});
|
||||
|
||||
test('controls: two lines far apart, or different files, stay new', () => {
|
||||
const lined = row('native', 1, { ...native, id: 'r-dx-native-1', line: 10 });
|
||||
expect(reconcile([lined], [row('outside', 1, { severity: 'High', title: 'retry loop has no bound', file: 'src/resolver.ts', line: 90 })]).rows[0]!.disposition).toBe('new');
|
||||
expect(reconcile([native], [row('outside', 1, { severity: 'High', title: 'CAP6:src/cli.ts:fix', file: 'src/cli.ts' })]).rows[0]!.disposition).toBe('new');
|
||||
});
|
||||
|
||||
test('a title twin is placed before the same-file fallback, so an unrelated finding cannot take its native', () => {
|
||||
const n1 = row('native', 1, { severity: 'High', title: 'token refresh race drops the session', file: 'plan.md' });
|
||||
const o1 = row('outside', 1, { severity: 'High', title: 'migration lacks rollback', file: 'plan.md' });
|
||||
const o2 = row('outside', 2, { severity: 'High', title: 'token refresh race drops the session', file: 'plan.md' });
|
||||
const { rows } = reconcile([n1], [o1, o2]);
|
||||
expect(rows.map(r => [r.outside_id, r.native_id, r.basis])).toEqual([[o1.id, null, 'none'], [o2.id, n1.id, 'title']]);
|
||||
});
|
||||
|
||||
test('control: the same CAP item and wording on two different files never pairs by title', () => {
|
||||
const a = row('native', 1, { severity: 'High', title: 'CAP6:bin/gstack-upgrade:recovery no retry bound on network failure', file: 'bin/gstack-upgrade' });
|
||||
const b = row('outside', 1, { severity: 'High', title: 'CAP6:lib/telemetry-sync.ts:recovery no retry bound on network failure', file: 'lib/telemetry-sync.ts' });
|
||||
expect(reconcile([a], [b]).rows[0]).toMatchObject({ disposition: 'new', native_id: null });
|
||||
});
|
||||
|
||||
test('two CAP titles on different paths never match by title, even when both findings sit on the plan file', () => {
|
||||
const A = 'CAP6:bin/gstack-eval-plan pilot start:fix no fix line names the command';
|
||||
const B = 'CAP6:bin/gstack-outside-voice run:fix no fix line names the command';
|
||||
expect(titleScore(A, B)).toBe(0);
|
||||
expect(titleScore(A, A)).toBe(1);
|
||||
expect(titleScore(A, 'no fix line names the command')).toBe(1);
|
||||
const n = row('native', 1, { severity: 'High', title: A, file: 'plan.md', resolution: 'name the command' });
|
||||
const o = row('outside', 1, { severity: 'High', title: B, file: 'plan.md', resolution: 'name the command' });
|
||||
expect(reconcile([n], [o]).rows[0]).toMatchObject({ basis: 'file', disposition: 'unverified' });
|
||||
const bare = (x: any) => { const { file, ...rest } = x; return rest; };
|
||||
expect(reconcile([bare(n)], [bare(o)]).rows[0]).toMatchObject({ disposition: 'new', native_id: null });
|
||||
});
|
||||
});
|
||||
@@ -206,6 +206,16 @@ describe('Autoplan parent publication guard', () => {
|
||||
expect(f.evaluate()).toMatchObject({ allow: false, reason: expect.stringContaining('earlier checkpoint') });
|
||||
});
|
||||
|
||||
test('#3651: a rerun checkpoint whose create result carries the cwd-reset footer still retires the old close', () => {
|
||||
const { f, tasks } = completedCycle(), create = checkpointWriter(f);
|
||||
const oldClose = f.events.find(e => e.kind === 'use' && e.toolUseId === 'eng-close')!;
|
||||
create('eng', 'new-generation');
|
||||
(f.events.at(-1) as any).content += '\nShell cwd was reset to /home/user/project';
|
||||
f.read('old-close-replayed', (oldClose as any).input.file_path as string); f.message('Phase 3 complete.');
|
||||
f.input.tool_input = { file_path: tasks }; f.current();
|
||||
expect(f.evaluate()).toMatchObject({ allow: false, reason: expect.stringContaining('earlier checkpoint') });
|
||||
});
|
||||
|
||||
for (const status of ['partial', 'failed', 'pending'] as const)
|
||||
test(`a ${status} rerun close cannot borrow the first cycle's completed report`, () => {
|
||||
const { f, create, tasks } = completedCycle(), rerun = create('eng', 'rerun');
|
||||
@@ -1596,3 +1606,56 @@ describe('PAX-3638: symlinked plan directory and superseded failed inits', () =>
|
||||
expect(f.evaluate()).toMatchObject({ allow: false, reason: expect.stringContaining('(code mutation_pending,') });
|
||||
});
|
||||
});
|
||||
|
||||
describe('#3651: Claude Code cwd-reset footer on a Bash result', () => {
|
||||
const FOOTER = '\nShell cwd was reset to /home/user/project';
|
||||
function withInitTail(tail: string) {
|
||||
const f = fixture();
|
||||
(f.events.find(e => e.kind === 'result' && e.toolUseId === 'init') as any).content += tail;
|
||||
return f;
|
||||
}
|
||||
const ready = (f: ReturnType<typeof fixture>) => { f.message(); f.current(); f.reorder(); return f.evaluate(); };
|
||||
|
||||
test('an init result carrying the footer still binds', () => {
|
||||
expect(ready(withInitTail(FOOTER))).toEqual({ allow: true });
|
||||
});
|
||||
|
||||
for (const [name, tail] of [['other trailing text', '\nunexpected trailing output'], ['two footers', `${FOOTER}${FOOTER}`], ['a footer that is not the last line', `${FOOTER}\n`]] as const)
|
||||
test(`${name} after an init result is init_failed with the re-run hint, not the generic evidence denial`, () => {
|
||||
const decision = ready(withInitTail(tail)) as any;
|
||||
expect(decision.allow).toBe(false);
|
||||
expect(decision.reason).toContain('(code init_failed,');
|
||||
expect(decision.reason).toContain('re-run');
|
||||
});
|
||||
|
||||
test('#3651: the hook path keeps the first binding when an init is re-run after a typed user message', async () => {
|
||||
const footer = '\nShell cwd was reset to /home/user/project';
|
||||
const run = async (tail: string, failed?: unknown) => {
|
||||
const f = fixture(); (f.events[1] as any).content += tail; f.message();
|
||||
if (failed !== undefined) { f.use('bad-init', 'Bash', (f.events[0] as any).input); f.result('bad-init', { content: failed }); }
|
||||
f.use('re-init', 'Bash', (f.events[0] as any).input); f.result('re-init', { content: JSON.stringify({ ...f.init, reused: true }) + tail });
|
||||
f.current();
|
||||
const { rows, record } = f.journal();
|
||||
const at = rows.findIndex(r => Array.isArray(r.message.content) && r.message.content.some((b: any) => b.type === 'tool_use' && ['bad-init', 're-init'].includes(b.id)));
|
||||
(rows[at - 1]!.message as any).stop_reason = 'end_turn';
|
||||
rows.splice(at, 0, record('user', 'Looks good, keep going.', { origin: { kind: 'human' }, promptSource: 'typed', promptId: randomUUID() }));
|
||||
rows.forEach((r, i) => { r.parentUuid = i ? rows[i - 1]!.uuid : null; });
|
||||
fs.writeFileSync(f.input.transcript_path, rows.map(x => JSON.stringify(x)).join('\n') + '\n');
|
||||
return withNativeProjectDirectory(f.cwd, () => runPublicationHook(f.input, ROOT));
|
||||
};
|
||||
expect(await run('')).toEqual(OWNED_READ_APPROVAL);
|
||||
expect(await run(footer)).toEqual(OWNED_READ_APPROVAL);
|
||||
expect(await run('', 'unexpected output')).toEqual(OWNED_READ_APPROVAL);
|
||||
expect(await run('', [{ type: 'text', text: 'a' }, { type: 'text', text: 'b' }])).toEqual(OWNED_READ_APPROVAL);
|
||||
});
|
||||
|
||||
for (const [name, content] of [['did not parse', undefined], ['had no single text block', [{ type: 'text', text: 'a' }, { type: 'text', text: 'b' }]]] as const)
|
||||
test(`a later clean init recovers the session after an init whose output ${name}`, () => {
|
||||
const f = withInitTail('\nunexpected trailing output');
|
||||
if (content) (f.events[1] as any).content = content;
|
||||
f.events.splice(2, 0, { ...(structuredClone(f.events[0]!) as any), toolUseId: 'again' },
|
||||
{ ...(structuredClone(f.events[1]!) as any), toolUseId: 'again', content: JSON.stringify({ ...f.init, reused: true }) });
|
||||
expect(ready(f)).toEqual({ allow: true });
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,213 @@
|
||||
/**
|
||||
* `gstack-autoplan run` with the api runner for both voices (GBRA-223 claim 6):
|
||||
* a complete autoplan result (INPUT receipt plus the findings fence) binds and
|
||||
* the run reaches its gate; a response stopped at the output limit is named
|
||||
* truncated and interrupts the run. The provider is a local server, so no key
|
||||
* or network is used.
|
||||
*/
|
||||
import { afterAll, beforeAll, describe, expect, test } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { DEFAULT_MAX_TOKENS } from '../lib/outside-voice-api';
|
||||
import { AUTOPLAN, ROOT, SMALL_PLAN, env } from './helpers/autoplan-run-fixture';
|
||||
|
||||
const TMP = fs.mkdtempSync(path.join(os.tmpdir(), 'autoplan-run-api-'));
|
||||
let mode: 'ok' | 'cut' | 'bad-outside' | 'bad-native' | 'hang' | 'down' = 'ok';
|
||||
let calls = 0;
|
||||
let openaiLimits: unknown[] = [];
|
||||
let server: ReturnType<typeof Bun.serve>;
|
||||
beforeAll(() => {
|
||||
server = Bun.serve({
|
||||
port: 0,
|
||||
async fetch(req) {
|
||||
calls++;
|
||||
if (mode === 'hang') { await Bun.sleep(5000); return new Response('late', { status: 504 }); }
|
||||
if (mode === 'down') return new Response('overloaded', { status: 503 });
|
||||
const body = await req.json();
|
||||
const prompt = String(body.messages[0].content);
|
||||
const m = /INPUT: ([a-z]+) ([0-9a-f]{64})/.exec(prompt.slice(prompt.lastIndexOf('RESULT FORMAT')))!;
|
||||
const text = mode === 'cut'
|
||||
? `INPUT: ${m[1]} ${m[2]}\n\nA long review that ran out of output tokens before the findings fen`
|
||||
: `INPUT: ${m[1]} ${m[2]}\n\nComplete review.\n\n\`\`\`gstack-findings\n{"id":"F1","severity":"Medium","title":"Name the retry metric","fix":"add retries_total"}\n\`\`\`\n`;
|
||||
if (new URL(req.url).pathname.endsWith('/messages')) return Response.json({ model: body.model, stop_reason: mode === 'cut' ? 'max_tokens' : 'end_turn', content: [{ type: 'text', text: mode === 'bad-native' ? `INPUT: ${m[1]} ${m[2]}\n\nA review with no findings fence.` : text }], usage: { input_tokens: 10, output_tokens: 20 } });
|
||||
openaiLimits.push(body.max_completion_tokens);
|
||||
return Response.json({ model: body.model, choices: [{ finish_reason: mode === 'cut' ? 'length' : 'stop', message: { content: mode === 'bad-outside' ? `INPUT: ${m[1]} ${m[2]}\n\nA review with no findings fence.` : text } }], usage: { prompt_tokens: 10, completion_tokens: 20 } });
|
||||
},
|
||||
});
|
||||
});
|
||||
afterAll(() => { server.stop(true); fs.rmSync(TMP, { recursive: true, force: true }); });
|
||||
|
||||
async function run(name: string, extra: string[] = [], keys: Record<string, string> = { ANTHROPIC_API_KEY: 'ak-test', OPENAI_API_KEY: 'sk-test' }, nativeModel = 'anthropic/claude-opus-5-5') {
|
||||
const dir = path.join(TMP, name);
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
fs.writeFileSync(path.join(dir, 'plan.md'), SMALL_PLAN);
|
||||
const out = path.join(dir, 'run');
|
||||
const base = `http://127.0.0.1:${server.port}`;
|
||||
const proc = Bun.spawn(['bun', AUTOPLAN, 'run', '--out', out, '--plan', path.join(dir, 'plan.md'), '--no-ui', '--no-dx', '--native-model', nativeModel, '--outside-model', 'openai/gpt-6-astra', '--runner', 'api', '--native-runner', 'api', ...extra], {
|
||||
cwd: ROOT, stdout: 'pipe', stderr: 'pipe',
|
||||
env: env({ ANTHROPIC_API_KEY: '', OPENAI_API_KEY: '', ...keys, ANTHROPIC_BASE_URL: base, OPENAI_BASE_URL: `${base}/v1`, GSTACK_AUTOPLAN_CHECKPOINT_DIR: '', GSTACK_STATE_ROOT: path.join(TMP, 'state') }),
|
||||
});
|
||||
const timer = setTimeout(() => proc.kill('SIGKILL'), 120_000);
|
||||
const [status, stdout, stderr] = await Promise.all([proc.exited, new Response(proc.stdout).text(), new Response(proc.stderr).text()]);
|
||||
clearTimeout(timer);
|
||||
return { status, stdout, stderr, out, state: fs.existsSync(path.join(out, 'run.json')) ? JSON.parse(fs.readFileSync(path.join(out, 'run.json'), 'utf8')) : null };
|
||||
}
|
||||
|
||||
describe('gstack-autoplan run --native-runner api --runner api', () => {
|
||||
test('a complete result binds both voices in every phase and the run reaches its gate', async () => {
|
||||
mode = 'ok';
|
||||
const r = await run('ok');
|
||||
expect(r.status, r.stderr).toBe(0);
|
||||
expect(r.stdout).toContain(`GSTACK_RESULT: skill=autoplan status=gate_pending run=${r.out}`);
|
||||
for (const phase of ['ceo', 'eng']) for (const voice of ['native', 'outside']) expect(r.state.phases[phase][voice]).toMatchObject({ status: 'completed', findings: 1 });
|
||||
}, 150_000);
|
||||
|
||||
test('a spend cap binds api attempts at their priced worst case and settles them at their usage', async () => {
|
||||
mode = 'ok';
|
||||
openaiLimits = [];
|
||||
const r = await run('capped', ['--spend-cap', '8']);
|
||||
expect(r.status, r.stderr).toBe(0);
|
||||
expect(r.stdout).toContain(`GSTACK_RESULT: skill=autoplan status=gate_pending run=${r.out}`);
|
||||
expect(r.stdout).not.toMatch(/^ESTIMATE:/m);
|
||||
const ledger = JSON.parse(fs.readFileSync(path.join(r.out, 'spend.json'), 'utf8'));
|
||||
expect(ledger.reserved).toEqual([]);
|
||||
expect(ledger.settled.length).toBe(4);
|
||||
for (const s of ledger.settled) expect(s.usd).toBeGreaterThan(0);
|
||||
expect(openaiLimits.length).toBeGreaterThan(0);
|
||||
for (const limit of openaiLimits) expect(limit).toBe(DEFAULT_MAX_TOKENS);
|
||||
calls = 0;
|
||||
const zero = await run('zero-cap', ['--spend-cap', '0']);
|
||||
expect(zero.status).toBe(3);
|
||||
expect(zero.stdout + zero.stderr).toContain('SPEND_CAP_EXCEEDED');
|
||||
expect(calls).toBe(0);
|
||||
}, 150_000);
|
||||
|
||||
test('an outside result with no findings fence leaves the run native-only instead of aborting it', async () => {
|
||||
mode = 'bad-outside';
|
||||
const r = await run('bad-outside');
|
||||
expect(r.status, r.stderr).toBe(0);
|
||||
expect(r.state.status).toBe('gate_pending');
|
||||
expect(r.state.phases.ceo.outside.status).toBe('unavailable');
|
||||
expect(r.state.phases.ceo.native).toMatchObject({ status: 'completed', findings: 1 });
|
||||
}, 150_000);
|
||||
|
||||
test('--max-tokens needs a positive integer', async () => {
|
||||
const r = await run('max0', ['--max-tokens', '0']);
|
||||
expect(r.status).toBe(2);
|
||||
expect(r.stderr).toContain('--max-tokens needs a positive integer');
|
||||
expect(r.state).toBeNull();
|
||||
}, 30_000);
|
||||
|
||||
test('a response stopped at the output limit is named truncated and interrupts the run', async () => {
|
||||
mode = 'cut';
|
||||
const r = await run('cut');
|
||||
expect(r.status).toBe(1);
|
||||
expect(r.stderr).toMatch(/native voice unavailable: truncated: .*output limit/);
|
||||
expect(r.state.status).toBe('interrupted');
|
||||
}, 150_000);
|
||||
|
||||
test('a truncated call settles what it cost, and a retry with a larger output limit is admitted afresh against the cap', async () => {
|
||||
mode = 'cut';
|
||||
const first = await run('cut-retry', ['--spend-cap', '8']);
|
||||
expect(first.state.status).toBe('interrupted');
|
||||
const truncated = first.state.phases.ceo.native.attempts[0].attempt;
|
||||
const ledger = JSON.parse(fs.readFileSync(path.join(first.out, 'spend.json'), 'utf8'));
|
||||
expect(ledger.reserved.map((r: any) => r.attempt)).not.toContain(truncated);
|
||||
expect(ledger.settled.find((s: any) => s.attempt === truncated).usd).toBeGreaterThan(0);
|
||||
mode = 'ok';
|
||||
calls = 0;
|
||||
const retry = await run('cut-retry', ['--spend-cap', '8', '--max-tokens', '2000000']);
|
||||
expect(retry.status).toBe(3);
|
||||
expect(retry.stdout + retry.stderr).toContain('SPEND_CAP_EXCEEDED');
|
||||
expect(calls).toBe(0);
|
||||
}, 150_000);
|
||||
|
||||
test('a call refused before any request leaves charges nothing: with no key the attempt stays open and the rerun reuses it at the cap that admitted it', async () => {
|
||||
mode = 'ok';
|
||||
calls = 0;
|
||||
const first = await run('no-key', ['--spend-cap', '8'], { OPENAI_API_KEY: 'sk-test' });
|
||||
expect(first.state.status).toBe('interrupted');
|
||||
expect(calls).toBe(0);
|
||||
const open = first.state.phases.ceo.native.attempts[0];
|
||||
expect(open.state).toBe('dispatched');
|
||||
expect(JSON.parse(fs.readFileSync(path.join(first.out, 'spend.json'), 'utf8')).settled).toEqual([]);
|
||||
const retry = await run('no-key', ['--spend-cap', '8']);
|
||||
expect(retry.status, retry.stderr).toBe(0);
|
||||
expect(retry.stdout).toContain(`GSTACK_RESULT: skill=autoplan status=gate_pending run=${retry.out}`);
|
||||
expect(retry.state.phases.ceo.native.attempts.map((a: any) => a.attempt)).toEqual([open.attempt]);
|
||||
const ledger = JSON.parse(fs.readFileSync(path.join(retry.out, 'spend.json'), 'utf8'));
|
||||
expect(ledger.settled.length).toBe(4);
|
||||
expect(ledger.released ?? []).toEqual([]);
|
||||
for (const s of ledger.settled) expect(s.label).toMatch(/^(ceo|eng)\/(native|outside)$/);
|
||||
}, 150_000);
|
||||
|
||||
test('a call that was sent and timed out with no usage is charged its reservation, so the cap binds on the rerun', async () => {
|
||||
mode = 'hang';
|
||||
const first = await run('timed-out', ['--spend-cap', '8', '--deadline', '0.02']);
|
||||
expect(first.state.status).toBe('interrupted');
|
||||
const sent = first.state.phases.ceo.native.attempts[0];
|
||||
expect(sent.state).toBe('superseded');
|
||||
const ledger = JSON.parse(fs.readFileSync(path.join(first.out, 'spend.json'), 'utf8'));
|
||||
expect(ledger.reserved.map((r: any) => r.attempt)).not.toContain(sent.attempt);
|
||||
expect(ledger.settled.find((s: any) => s.attempt === sent.attempt).usd).toBe('unknown');
|
||||
}, 150_000);
|
||||
|
||||
test('a sent call the provider refused with a 503 is charged its reservation', async () => {
|
||||
mode = 'down';
|
||||
const r = await run('refused-503', ['--spend-cap', '8']);
|
||||
expect(r.state.status).toBe('interrupted');
|
||||
const sent = r.state.phases.ceo.native.attempts[0];
|
||||
expect(sent.state).toBe('superseded');
|
||||
const ledger = JSON.parse(fs.readFileSync(path.join(r.out, 'spend.json'), 'utf8'));
|
||||
expect(ledger.reserved.map((x: any) => x.attempt)).not.toContain(sent.attempt);
|
||||
expect(ledger.settled.find((s: any) => s.attempt === sent.attempt).usd).toBe('unknown');
|
||||
}, 150_000);
|
||||
|
||||
test('a paid native call whose result submit refuses is settled at its usage, so the rerun is admitted afresh', async () => {
|
||||
mode = 'bad-native';
|
||||
const r = await run('refused-result', ['--spend-cap', '8']);
|
||||
expect(r.status).not.toBe(0);
|
||||
const sent = r.state.phases.ceo.native.attempts[0];
|
||||
expect(sent.state).toBe('superseded');
|
||||
const ledger = JSON.parse(fs.readFileSync(path.join(r.out, 'spend.json'), 'utf8'));
|
||||
expect(ledger.reserved.map((x: any) => x.attempt)).not.toContain(sent.attempt);
|
||||
expect(ledger.settled.find((s: any) => s.attempt === sent.attempt).usd).toBeGreaterThan(0);
|
||||
}, 150_000);
|
||||
|
||||
test('a call in flight when the process died is charged its reservation on the next run, which dispatches a fresh attempt', async () => {
|
||||
mode = 'hang';
|
||||
calls = 0;
|
||||
const dir = path.join(TMP, 'killed');
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
fs.writeFileSync(path.join(dir, 'plan.md'), SMALL_PLAN);
|
||||
const out = path.join(dir, 'run');
|
||||
const base = `http://127.0.0.1:${server.port}`;
|
||||
const proc = Bun.spawn(['bun', AUTOPLAN, 'run', '--out', out, '--plan', path.join(dir, 'plan.md'), '--no-ui', '--no-dx', '--native-model', 'anthropic/claude-opus-5-5', '--outside-model', 'openai/gpt-6-astra', '--runner', 'api', '--native-runner', 'api', '--spend-cap', '8'], {
|
||||
cwd: ROOT, stdout: 'ignore', stderr: 'ignore',
|
||||
env: env({ ANTHROPIC_API_KEY: 'ak-test', OPENAI_API_KEY: 'sk-test', ANTHROPIC_BASE_URL: base, OPENAI_BASE_URL: `${base}/v1`, GSTACK_AUTOPLAN_CHECKPOINT_DIR: '', GSTACK_STATE_ROOT: path.join(TMP, 'state') }),
|
||||
});
|
||||
for (let i = 0; i < 200 && calls === 0; i++) await Bun.sleep(50);
|
||||
expect(calls).toBe(1);
|
||||
proc.kill('SIGKILL');
|
||||
await proc.exited;
|
||||
mode = 'ok';
|
||||
const retry = await run('killed', ['--spend-cap', '8']);
|
||||
expect(retry.status, retry.stderr).toBe(0);
|
||||
const [lost, fresh] = retry.state.phases.ceo.native.attempts;
|
||||
expect(lost.state).toBe('superseded');
|
||||
expect(fresh.state).toBe('submitted');
|
||||
const ledger = JSON.parse(fs.readFileSync(path.join(retry.out, 'spend.json'), 'utf8'));
|
||||
expect(ledger.settled.find((s: any) => s.attempt === lost.attempt).usd).toBe('unknown');
|
||||
}, 150_000);
|
||||
|
||||
test('with a spend cap an unpriced api model reserves the default in any session kind, so a cap below it refuses before any call', async () => {
|
||||
mode = 'ok';
|
||||
calls = 0;
|
||||
const kind = ['GSTACK', 'SESSION', 'KIND'].join('_');
|
||||
const r = await run('unpriced', ['--spend-cap', '1'], { ANTHROPIC_API_KEY: 'ak-test', OPENAI_API_KEY: 'sk-test', [kind]: 'spawned' }, 'anthropic/claude-haiku-5-5');
|
||||
expect(r.status).toBe(3);
|
||||
expect(r.stdout + r.stderr).toContain('SPEND_CAP_EXCEEDED');
|
||||
expect(calls).toBe(0);
|
||||
}, 150_000);
|
||||
});
|
||||
@@ -106,7 +106,8 @@ describe('next/submit loop', () => {
|
||||
expect(v.status, v.stdout + v.stderr).toBe(0);
|
||||
expect(JSON.parse(v.stdout)).toMatchObject({ valid: true, status: 'gate_pending' });
|
||||
const gate = JSON.parse(fs.readFileSync(path.join(out, 'gate.json'), 'utf8'));
|
||||
expect(gate.items.map((i: any) => i.id)).toEqual(['uc1', 'uc2', 'p1']);
|
||||
expect(gate.items.map((i: any) => i.id)).toEqual(['uc1', 'd1', 'd2', 'uc2', 'd3', 'd4', 'p1']);
|
||||
expect(gate.items.filter((i: any) => i.finding).map((i: any) => i.title.split(':')[0])).toEqual(['Critical', 'High', 'Critical', 'High']);
|
||||
const decisions = fs.readFileSync(path.join(out, 'decisions.jsonl'), 'utf8').trim().split('\n').map(l => JSON.parse(l));
|
||||
expect(decisions.every((d: any) => d.status === 'pending' && d.gate_rev === 1)).toBe(true);
|
||||
const tasks = fs.readFileSync(path.join(out, 'tasks.jsonl'), 'utf8').trim().split('\n').map(l => JSON.parse(l));
|
||||
|
||||
+150
-26
@@ -843,7 +843,7 @@ describe('gstack-brain-sync per-file hold', () => {
|
||||
}
|
||||
function setup() {
|
||||
init();
|
||||
write(SHARED, '# shared v1\n');
|
||||
write(SHARED, '{"note":"shared v1"}\n');
|
||||
write(OTHER, '# other v1\n');
|
||||
expect(drain().status).toBe(0);
|
||||
expect(statusJson().status).toBe('ok');
|
||||
@@ -856,15 +856,15 @@ describe('gstack-brain-sync per-file hold', () => {
|
||||
|
||||
test('a dirty tracked file the remote did not touch rides through the merge', () => {
|
||||
setup();
|
||||
peerCommit(SHARED, '# shared from the peer\n');
|
||||
peerCommit(SHARED, '{"note":"shared from the peer"}\n');
|
||||
write(OTHER, '# other, local edit not queued\n', false);
|
||||
write(CLEAN, '# a plan\n');
|
||||
expect(drain().status).toBe(0);
|
||||
expect(statusJson().status).toBe('ok');
|
||||
expect(remoteFiles()).toContain(CLEAN);
|
||||
expect(remoteShow(SHARED).stdout).toBe('# shared from the peer\n');
|
||||
expect(remoteShow(SHARED).stdout).toBe('{"note":"shared from the peer"}\n');
|
||||
expect(fs.readFileSync(path.join(tmpHome, OTHER), 'utf-8')).toBe('# other, local edit not queued\n');
|
||||
expect(fs.readFileSync(path.join(tmpHome, SHARED), 'utf-8')).toBe('# shared from the peer\n');
|
||||
expect(fs.readFileSync(path.join(tmpHome, SHARED), 'utf-8')).toBe('{"note":"shared from the peer"}\n');
|
||||
});
|
||||
|
||||
/** The restore command the status names, run as printed. */
|
||||
@@ -878,17 +878,17 @@ describe('gstack-brain-sync per-file hold', () => {
|
||||
|
||||
test('a dirty tracked file the remote also changed no longer leaves the push failed', () => {
|
||||
setup();
|
||||
peerCommit(SHARED, '# shared from the peer\n');
|
||||
write(SHARED, '# shared, local edit not queued\n', false);
|
||||
peerCommit(SHARED, '{"note":"shared from the peer"}\n');
|
||||
write(SHARED, '{"note":"shared, local edit not queued"}\n', false);
|
||||
write(CLEAN, '# a plan\n');
|
||||
expect(drain().status).toBe(0);
|
||||
const s = statusJson();
|
||||
expect(s.status).toBe('ok');
|
||||
expect(remoteFiles()).toContain(CLEAN);
|
||||
expect(remoteShow(SHARED).stdout).toBe('# shared from the peer\n');
|
||||
expect(remoteShow(SHARED).stdout).toBe('{"note":"shared from the peer"}\n');
|
||||
// The merged version stays in the working tree: no conflict markers,
|
||||
// nothing staged, and nothing stale for a later sync to publish.
|
||||
expect(fs.readFileSync(path.join(tmpHome, SHARED), 'utf-8')).toBe('# shared from the peer\n');
|
||||
expect(fs.readFileSync(path.join(tmpHome, SHARED), 'utf-8')).toBe('{"note":"shared from the peer"}\n');
|
||||
expect(unmerged()).toBe('');
|
||||
expect(staged()).toBe('');
|
||||
expect(s.message).toContain(`local edits to ${SHARED} conflicted`);
|
||||
@@ -900,23 +900,23 @@ describe('gstack-brain-sync per-file hold', () => {
|
||||
write(OTHER, '# other v2\n');
|
||||
expect(drain().status).toBe(0);
|
||||
expect(statusJson().status).toBe('ok');
|
||||
expect(remoteShow(SHARED).stdout).toBe('# shared from the peer\n');
|
||||
expect(git(['show', `${sha}:${SHARED}`]).stdout).toBe('# shared, local edit not queued\n');
|
||||
expect(remoteShow(SHARED).stdout).toBe('{"note":"shared from the peer"}\n');
|
||||
expect(git(['show', `${sha}:${SHARED}`]).stdout).toBe('{"note":"shared, local edit not queued"}\n');
|
||||
// The printed command brings the local edit back.
|
||||
expect(recover(s.message)).toBe(sha!);
|
||||
expect(fs.readFileSync(path.join(tmpHome, SHARED), 'utf-8')).toBe('# shared, local edit not queued\n');
|
||||
expect(fs.readFileSync(path.join(tmpHome, SHARED), 'utf-8')).toBe('{"note":"shared, local edit not queued"}\n');
|
||||
});
|
||||
|
||||
test('a local deletion of a file the remote changed keeps the remote file and stashes the deletion', () => {
|
||||
setup();
|
||||
peerCommit(SHARED, '# shared from the peer\n');
|
||||
peerCommit(SHARED, '{"note":"shared from the peer"}\n');
|
||||
fs.rmSync(path.join(tmpHome, SHARED));
|
||||
write(CLEAN, '# a plan\n');
|
||||
expect(drain().status).toBe(0);
|
||||
const s = statusJson();
|
||||
expect(s.status).toBe('ok');
|
||||
expect(remoteShow(SHARED).stdout).toBe('# shared from the peer\n');
|
||||
expect(fs.readFileSync(path.join(tmpHome, SHARED), 'utf-8')).toBe('# shared from the peer\n');
|
||||
expect(remoteShow(SHARED).stdout).toBe('{"note":"shared from the peer"}\n');
|
||||
expect(fs.readFileSync(path.join(tmpHome, SHARED), 'utf-8')).toBe('{"note":"shared from the peer"}\n');
|
||||
expect(unmerged()).toBe('');
|
||||
expect(s.message).toContain(`local edits to ${SHARED} conflicted`);
|
||||
recover(s.message);
|
||||
@@ -934,7 +934,7 @@ describe('gstack-brain-sync per-file hold', () => {
|
||||
} finally {
|
||||
fs.rmSync(peer, { recursive: true, force: true });
|
||||
}
|
||||
write(SHARED, '# shared, local edit not queued\n', false);
|
||||
write(SHARED, '{"note":"shared, local edit not queued"}\n', false);
|
||||
write(CLEAN, '# a plan\n');
|
||||
expect(drain().status).toBe(0);
|
||||
const s = statusJson();
|
||||
@@ -943,7 +943,7 @@ describe('gstack-brain-sync per-file hold', () => {
|
||||
expect(fs.existsSync(path.join(tmpHome, SHARED))).toBe(false);
|
||||
expect(unmerged()).toBe('');
|
||||
recover(s.message);
|
||||
expect(fs.readFileSync(path.join(tmpHome, SHARED), 'utf-8')).toBe('# shared, local edit not queued\n');
|
||||
expect(fs.readFileSync(path.join(tmpHome, SHARED), 'utf-8')).toBe('{"note":"shared, local edit not queued"}\n');
|
||||
});
|
||||
|
||||
test('a conflicting file whose name looks like a pattern resets only itself', () => {
|
||||
@@ -954,16 +954,16 @@ describe('gstack-brain-sync per-file hold', () => {
|
||||
expect(git(['merge', '-q', '--ff-only', 'origin/main']).status).toBe(0);
|
||||
peerCommit(GLOB, '# glob from the peer\n');
|
||||
write(GLOB, '# glob, local edit not queued\n', false);
|
||||
write(SHARED, '# shared, local edit not queued\n', false);
|
||||
write(SHARED, '{"note":"shared, local edit not queued"}\n', false);
|
||||
write(CLEAN, '# a plan\n');
|
||||
expect(drain().status).toBe(0);
|
||||
const s = statusJson();
|
||||
expect(s.status).toBe('ok');
|
||||
expect(fs.readFileSync(path.join(tmpHome, GLOB), 'utf-8')).toBe('# glob from the peer\n');
|
||||
expect(fs.readFileSync(path.join(tmpHome, SHARED), 'utf-8')).toBe('# shared, local edit not queued\n');
|
||||
expect(fs.readFileSync(path.join(tmpHome, SHARED), 'utf-8')).toBe('{"note":"shared, local edit not queued"}\n');
|
||||
recover(s.message);
|
||||
expect(fs.readFileSync(path.join(tmpHome, GLOB), 'utf-8')).toBe('# glob, local edit not queued\n');
|
||||
expect(fs.readFileSync(path.join(tmpHome, SHARED), 'utf-8')).toBe('# shared, local edit not queued\n');
|
||||
expect(fs.readFileSync(path.join(tmpHome, SHARED), 'utf-8')).toBe('{"note":"shared, local edit not queued"}\n');
|
||||
});
|
||||
|
||||
test('a file that never checks out clean does not strand the other local edits', () => {
|
||||
@@ -973,7 +973,7 @@ describe('gstack-brain-sync per-file hold', () => {
|
||||
expect(drain().status).toBe(0);
|
||||
fs.mkdirSync(path.join(tmpHome, '.git', 'info'), { recursive: true });
|
||||
fs.writeFileSync(path.join(tmpHome, '.git', 'info', 'attributes'), '*.md text eol=lf\n');
|
||||
peerCommit(SHARED, '# shared from the peer\n');
|
||||
peerCommit(SHARED, '{"note":"shared from the peer"}\n');
|
||||
write(OTHER, '# other, local edit not queued\n', false);
|
||||
write(CLEAN, '# a plan\n');
|
||||
expect(drain().status).toBe(0);
|
||||
@@ -987,28 +987,28 @@ describe('gstack-brain-sync per-file hold', () => {
|
||||
setup();
|
||||
write(OTHER, '# other, older stashed work\n', false);
|
||||
expect(git(['stash', 'push', '-q', '-m', 'older work']).status).toBe(0);
|
||||
peerCommit(SHARED, '# shared from the peer\n');
|
||||
write(SHARED, '# shared, local edit not queued\n', false);
|
||||
peerCommit(SHARED, '{"note":"shared from the peer"}\n');
|
||||
write(SHARED, '{"note":"shared, local edit not queued"}\n', false);
|
||||
write(CLEAN, '# a plan\n');
|
||||
const shim = gitShim('case " $* " in *" stash push "*) exit 0;; esac');
|
||||
expect(drain(shim.env).status).toBe(0);
|
||||
expect(statusJson().status).toBe('push_failed');
|
||||
expect(git(['stash', 'list']).stdout).toContain('older work');
|
||||
expect(fs.readFileSync(path.join(tmpHome, OTHER), 'utf-8')).toBe('# other v1\n');
|
||||
expect(fs.readFileSync(path.join(tmpHome, SHARED), 'utf-8')).toBe('# shared, local edit not queued\n');
|
||||
expect(fs.readFileSync(path.join(tmpHome, SHARED), 'utf-8')).toBe('{"note":"shared, local edit not queued"}\n');
|
||||
});
|
||||
|
||||
test('a conflicting merge is aborted and git\'s reason reaches the status', () => {
|
||||
setup();
|
||||
peerCommit(SHARED, '# shared from the peer\n');
|
||||
write(SHARED, '# shared, local commit\n');
|
||||
peerCommit(SHARED, '{"note":"shared from the peer"}\n');
|
||||
write(SHARED, '{"note":"shared, local commit"}\n');
|
||||
expect(drain().status).toBe(0);
|
||||
const s = statusJson();
|
||||
expect(s.status).toBe('push_failed');
|
||||
expect(s.message).toContain(`Merge conflict in ${SHARED}`);
|
||||
expect(mergeHead()).toBe(false);
|
||||
expect(unmerged()).toBe('');
|
||||
expect(fs.readFileSync(path.join(tmpHome, SHARED), 'utf-8')).toBe('# shared, local commit\n');
|
||||
expect(fs.readFileSync(path.join(tmpHome, SHARED), 'utf-8')).toBe('{"note":"shared, local commit"}\n');
|
||||
// The next drain is not blocked by a half-finished merge.
|
||||
write(OTHER, '# other v2\n');
|
||||
expect(drain().status).toBe(0);
|
||||
@@ -1270,6 +1270,130 @@ describe('gstack-brain-sync per-file hold', () => {
|
||||
expect(remoteShow(SNAP).stdout).toContain('"ccc"');
|
||||
});
|
||||
});
|
||||
|
||||
describe('a synced JSON file that does not parse is never committed (#3763)', () => {
|
||||
const PROFILE = 'developer-profile.json';
|
||||
const MARKED = '{\n "sessions": [],\n<<<<<<< HEAD\n "v": "local"\n=======\n "v": "peer"\n>>>>>>> origin/main\n}\n';
|
||||
const parses = (text: string) => { try { JSON.parse(text); return true; } catch { return false; } };
|
||||
|
||||
test('a queued profile holding conflict markers is held back; the rest syncs; a repaired profile syncs next', () => {
|
||||
init();
|
||||
write(PROFILE, MARKED);
|
||||
write(CLEAN, '# a plan\n');
|
||||
expect(drain().status).toBe(0);
|
||||
expect(remoteFiles()).toContain(CLEAN);
|
||||
expect(remoteFiles()).not.toContain(PROFILE);
|
||||
expect(staged()).toBe('');
|
||||
expect(spoolText()).toContain(PROFILE);
|
||||
const s = statusJson();
|
||||
expect(s.status).toBe('held');
|
||||
expect(s.held.map((h: { path: string; rule: string }) => [h.path, h.rule])).toEqual([[PROFILE, 'invalid-json']]);
|
||||
expect(s.held[0].fixes[0]).toContain(`${PROFILE} is not valid JSON`);
|
||||
expect(s.held[0].fixes).toHaveLength(1);
|
||||
expect(s.message).toContain('not valid JSON');
|
||||
expect(s.message).not.toContain('secret scan');
|
||||
|
||||
write(PROFILE, '{"sessions":[],"v":"repaired"}\n', false);
|
||||
expect(drain().status).toBe(0);
|
||||
expect(statusJson().status).toBe('ok');
|
||||
expect(remoteShow(PROFILE).stdout).toBe('{"sessions":[],"v":"repaired"}\n');
|
||||
});
|
||||
|
||||
test('a marker-free syntax error is held too', () => {
|
||||
init();
|
||||
write(PROFILE, '{"sessions": [],}\n');
|
||||
expect(drain().status).toBe(0);
|
||||
expect(remoteFiles()).not.toContain(PROFILE);
|
||||
expect(statusJson().held.map((h: { rule: string }) => h.rule)).toEqual(['invalid-json']);
|
||||
expect(spoolText()).toContain(PROFILE);
|
||||
});
|
||||
|
||||
test('an unparseable decisions.active.json holds its coupled decisions.jsonl until repaired', () => {
|
||||
init();
|
||||
writeDecisions(['a'], ['a']);
|
||||
expect(drain().status).toBe(0);
|
||||
write(LOG, [decide('a'), decide('bb')].join('\n') + '\n', false);
|
||||
write(SNAP, '[{"id":"a"},\n<<<<<<< HEAD\n{"id":"bb"}\n=======\n{"id":"c"}\n>>>>>>> origin/main\n]\n', false);
|
||||
run(['gstack-brain-enqueue', LOG]);
|
||||
expect(drain().status).toBe(0);
|
||||
const s = statusJson();
|
||||
expect(s.status).toBe('held');
|
||||
expect(s.held.map((h: { path: string; rule: string; dependents: string[] }) => [h.path, h.rule, h.dependents])).toEqual([[SNAP, 'invalid-json', [LOG]]]);
|
||||
expect(remoteShow(LOG).stdout).toBe(decide('a') + '\n');
|
||||
expect(spoolText()).toContain(LOG);
|
||||
|
||||
writeDecisions(['a', 'bb'], ['a', 'bb']);
|
||||
expect(drain().status).toBe(0);
|
||||
expect(statusJson().status).toBe('ok');
|
||||
expect(remoteShow(LOG).stdout).toBe([decide('a'), decide('bb')].join('\n') + '\n');
|
||||
});
|
||||
|
||||
test('two machines editing the profile never put conflict markers on the remote', () => {
|
||||
init();
|
||||
write(PROFILE, '{"sessions":[],"v":"base"}\n');
|
||||
expect(drain().status).toBe(0);
|
||||
// The retry merge commits with the ambient identity; a CI runner has none.
|
||||
expect(git(['config', 'user.name', 'fixture']).status).toBe(0);
|
||||
expect(git(['config', 'user.email', 'fixture@example.com']).status).toBe(0);
|
||||
const peer = fs.mkdtempSync(path.join(os.tmpdir(), 'brain-sync-peer-'));
|
||||
try {
|
||||
expect(git(['clone', '-q', bareRemote, peer]).status).toBe(0);
|
||||
fs.writeFileSync(path.join(peer, PROFILE), '{"sessions":[],"v":"peer"}\n');
|
||||
expect(git(['add', '-f', '--', PROFILE], peer).status).toBe(0);
|
||||
expect(git(['-c', 'user.name=peer', '-c', 'user.email=peer@example.com', 'commit', '-qm', 'peer profile'], peer).status).toBe(0);
|
||||
expect(git(['push', '-q', 'origin', 'HEAD'], peer).status).toBe(0);
|
||||
} finally {
|
||||
fs.rmSync(peer, { recursive: true, force: true });
|
||||
}
|
||||
write(PROFILE, '{"sessions":[],"v":"local"}\n');
|
||||
expect(drain().status).toBe(0);
|
||||
// Every later profile write queues the file again.
|
||||
run(['gstack-brain-enqueue', PROFILE]);
|
||||
write(CLEAN, '# a plan\n');
|
||||
expect(drain().status).toBe(0);
|
||||
expect(parses(remoteShow(PROFILE).stdout)).toBe(true);
|
||||
expect(remoteShow(PROFILE).stdout).not.toContain('<<<<<<<');
|
||||
expect(remoteShow(PROFILE).stdout).toBe('{"sessions":[],"v":"peer"}\n');
|
||||
});
|
||||
|
||||
test('an unfinished merge with a held profile commits nothing until the profile is repaired', () => {
|
||||
init();
|
||||
write(PROFILE, '{"sessions":[],"v":"base"}\n');
|
||||
expect(drain().status).toBe(0);
|
||||
expect(git(['config', 'user.name', 'fixture']).status).toBe(0);
|
||||
expect(git(['config', 'user.email', 'fixture@example.com']).status).toBe(0);
|
||||
const peer = fs.mkdtempSync(path.join(os.tmpdir(), 'brain-sync-peer-'));
|
||||
try {
|
||||
expect(git(['clone', '-q', bareRemote, peer]).status).toBe(0);
|
||||
fs.writeFileSync(path.join(peer, PROFILE), '{"sessions":[],"v":"peer"}\n');
|
||||
expect(git(['add', '-f', '--', PROFILE], peer).status).toBe(0);
|
||||
expect(git(['-c', 'user.name=peer', '-c', 'user.email=peer@example.com', 'commit', '-qm', 'peer profile'], peer).status).toBe(0);
|
||||
expect(git(['push', '-q', 'origin', 'HEAD'], peer).status).toBe(0);
|
||||
} finally {
|
||||
fs.rmSync(peer, { recursive: true, force: true });
|
||||
}
|
||||
fs.writeFileSync(path.join(tmpHome, PROFILE), '{"sessions":[],"v":"local"}\n');
|
||||
expect(git(['commit', '-qam', 'local profile']).status).toBe(0);
|
||||
expect(git(['fetch', '-q', 'origin']).status).toBe(0);
|
||||
const branch = git(['rev-parse', '--abbrev-ref', 'HEAD']).stdout.trim();
|
||||
expect(git(['merge', '--no-edit', `origin/${branch}`]).status).not.toBe(0);
|
||||
run(['gstack-brain-enqueue', PROFILE]);
|
||||
write(CLEAN, '# a plan\n');
|
||||
expect(drain().status).toBe(0);
|
||||
expect(statusJson().status).toBe('blocked');
|
||||
expect(statusJson().message).toContain('unfinished');
|
||||
expect(statusJson().message).not.toContain('--skip-file <path>');
|
||||
expect(remoteShow(PROFILE).stdout).toBe('{"sessions":[],"v":"peer"}\n');
|
||||
expect(remoteFiles()).not.toContain(CLEAN);
|
||||
expect(git(['rev-parse', '-q', '--verify', 'MERGE_HEAD']).status).toBe(0);
|
||||
|
||||
write(PROFILE, '{"sessions":[],"v":"merged"}\n', false);
|
||||
expect(drain().status).toBe(0);
|
||||
expect(statusJson().status).toBe('ok');
|
||||
expect(remoteShow(PROFILE).stdout).toBe('{"sessions":[],"v":"merged"}\n');
|
||||
expect(remoteFiles()).toContain(CLEAN);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
|
||||
@@ -11,7 +11,7 @@ import { afterAll, describe, expect, test } from 'bun:test';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { ROOT, resultOf } from './helpers/autoplan-run-fixture';
|
||||
import { ROOT, recCarrying, resultOf } from './helpers/autoplan-run-fixture';
|
||||
import { capyRunner, phaseVia, writePlan } from './helpers/autoplan-capy';
|
||||
|
||||
const r = capyRunner();
|
||||
@@ -69,7 +69,8 @@ describe('Capy Captain flow', () => {
|
||||
expect(j.brief).toStartWith('**Decision needed:** approve the plan "Retry backoff for the webhook worker".');
|
||||
for (const h of ['### Before → after', '### Why it matters', '### What caused this work', '### Big decisions', '### What we need from you']) expect(j.brief).toContain(h);
|
||||
|
||||
const answer = r.ap(['answer', '--out', out, '--gate-rev', String(j.gate_rev), '--reply', 'rec', '--rec-hash', j.rec_hash, '--by', 'agent:GSTA-1', '--owner-reply', 'rec']);
|
||||
const reply = recCarrying(out, 'rec');
|
||||
const answer = r.ap(['answer', '--out', out, '--gate-rev', String(j.gate_rev), '--reply', reply, '--rec-hash', j.rec_hash, '--by', 'agent:GSTA-1', '--owner-reply', reply]);
|
||||
expect(answer.status, answer.stderr + answer.stdout).toBe(0);
|
||||
expect(answer.stdout).toMatch(/^APPROVED: gate_rev=1/m);
|
||||
|
||||
@@ -84,7 +85,7 @@ describe('Capy Captain flow', () => {
|
||||
expect(bin('gstack-approvals', ['import', '--autoplan', out, '--by', 'Garry'], env, repo).stdout).toMatch(/^APPROVAL_IMPORTED: n=0 skipped=1 /);
|
||||
const table = bin('gstack-approvals', ['render'], env, repo).stdout;
|
||||
expect(table).toContain('Approve the plan "Retry backoff for the webhook worker"');
|
||||
expect(table).toContain('“rec” (Garry, relayed by agent:GSTA-1) | approved |');
|
||||
expect(table).toContain(`“${reply}” (Garry, relayed by agent:GSTA-1) | approved |`);
|
||||
expect(fs.readdirSync(path.join(mirror, 'acme-worker')).length).toBe(1);
|
||||
});
|
||||
|
||||
|
||||
@@ -21,8 +21,8 @@ describe('dx-error-contract-review grader (free controls for the paid case)', ()
|
||||
});
|
||||
|
||||
test('the grader keys on the title and severity shapes the product prompt asks for', () => {
|
||||
expect(DX_ERROR_COMPACT).toContain('as a Critical finding titled "CAP4:<path>:<item>"');
|
||||
expect(DX_ERROR_COMPACT).toContain('as a High finding titled "CAP6:<path>:<item>"');
|
||||
expect(DX_ERROR_COMPACT).toContain('as a Critical finding titled "CAP4:<path>:<item> <what fails>"');
|
||||
expect(DX_ERROR_COMPACT).toContain('as a High finding titled "CAP6:<path>:<item> <what fails>"');
|
||||
const rows = DX_ERROR_ITEMS.map((item, index) => ({ id: `F${index}`, severity: 'High', title: `CAP6:P1:${item} gap` }));
|
||||
expect(capFindings(fence(rows)).map(cap => cap.item)).toEqual([...DX_ERROR_ITEMS]);
|
||||
});
|
||||
|
||||
@@ -20,6 +20,8 @@ describe('agent-ready error contract reaches every DX reviewer', () => {
|
||||
for (const text of [DX_ERROR_CONTRACT, DX_ERROR_COMPACT]) {
|
||||
expect(text).toContain('CAP4:<path>:<item>');
|
||||
expect(text).toContain('CAP6:<path>:<item>');
|
||||
expect(text).toContain('CAP4:<path>:<item> <what fails>');
|
||||
expect(text).toContain('CAP6:<path>:<item> <what fails>');
|
||||
expect(text).toContain('4/10');
|
||||
expect(text).toContain('6/10');
|
||||
expect(text).toMatch(/[Uu]sage (and|or) validation error/);
|
||||
|
||||
@@ -141,7 +141,7 @@ test('live periodic census fits the declared CI wall including setup', () => {
|
||||
|
||||
test('registered allocation is deterministic and preserves every discovered file', () => {
|
||||
const files = collectPaidTestFiles();
|
||||
expect(files).toHaveLength(114); // + skill-e2e-planning-wave (GSTA-228); + skill-e2e-model-panel (GSTA-173); + skill-e2e-write-skill (GSTA-133); + skill-e2e-ship-frozen-scope (frozen review scope, GSTA-113 PR 2); + skill-e2e-failure-routing (#3157/#3158); + skill-e2e-dx-error-contract (#3226); + skill-e2e-gate-integrity (HONEST_WORK_GATE_INTEGRITY W8); + skill-e2e-autoplan-schema-canary, -guard-pty and -long-session (Oct 7 wave, CEO-13, ENG-18, CEO-15); + skill-e2e-ship-measure-seeded-flake (v1.91.32.0); W5a: 15 carve wrappers became one case-sharded file; + codex-e2e-multiblock-live and skill-e2e-design-model-smoke (v1.91.30.0);
|
||||
expect(files).toHaveLength(115); // + skill-e2e-autoplan-ceo-only (GST-537); + skill-e2e-planning-wave (GSTA-228); + skill-e2e-model-panel (GSTA-173); + skill-e2e-write-skill (GSTA-133); + skill-e2e-ship-frozen-scope (frozen review scope, GSTA-113 PR 2); + skill-e2e-failure-routing (#3157/#3158); + skill-e2e-dx-error-contract (#3226); + skill-e2e-gate-integrity (HONEST_WORK_GATE_INTEGRITY W8); + skill-e2e-autoplan-schema-canary, -guard-pty and -long-session (Oct 7 wave, CEO-13, ENG-18, CEO-15); + skill-e2e-ship-measure-seeded-flake (v1.91.32.0); W5a: 15 carve wrappers became one case-sharded file; + codex-e2e-multiblock-live and skill-e2e-design-model-smoke (v1.91.30.0);
|
||||
expect(files).toContain('test/skill-e2e-ship-skip.test.ts');
|
||||
const m = livePlan(files);
|
||||
expect(livePlan([...files].reverse())).toEqual(m);
|
||||
|
||||
Vendored
+1
-1
@@ -25,7 +25,7 @@
|
||||
"diagram": 4286,
|
||||
"document-generate": 6013,
|
||||
"document-release": 4454,
|
||||
"eval-plan": 5458,
|
||||
"eval-plan": 5107,
|
||||
"freeze": 1115,
|
||||
"gstack": 5350,
|
||||
"gstack-upgrade": 6297,
|
||||
|
||||
+2
-2
@@ -210,8 +210,8 @@ not a commit ID. Use `git diff <old-tree> <new-tree>` to compare these snapshots
|
||||
**Review policy** (docs/review-policy.md): only a finding that would fail a test,
|
||||
break production, or lose data or secrets blocks a pass; the rest is a PR-body
|
||||
note. Step 9's plan sets both reviews' depth (`--depth-floor standard|full` only
|
||||
raises it). Pass 1 reads the diff, later passes the range; the cap is two full
|
||||
passes and a fix-check. `NO FINDINGS` is complete. `ship: review_scope` (Step 3)
|
||||
raises it). Pass 1 reads the diff, later passes the range; the cap is two rounds,
|
||||
fix-checks included. `NO FINDINGS` is complete. `ship: review_scope` (Step 3)
|
||||
is the knob; the receipt's `review_loop` records the outcome.
|
||||
|
||||
### Ship control flow
|
||||
|
||||
+2
-2
@@ -210,8 +210,8 @@ not a commit ID. Use `git diff <old-tree> <new-tree>` to compare these snapshots
|
||||
**Review policy** (docs/review-policy.md): only a finding that would fail a test,
|
||||
break production, or lose data or secrets blocks a pass; the rest is a PR-body
|
||||
note. Step 9's plan sets both reviews' depth (`--depth-floor standard|full` only
|
||||
raises it). Pass 1 reads the diff, later passes the range; the cap is two full
|
||||
passes and a fix-check. `NO FINDINGS` is complete. `ship: review_scope` (Step 3)
|
||||
raises it). Pass 1 reads the diff, later passes the range; the cap is two rounds,
|
||||
fix-checks included. `NO FINDINGS` is complete. `ship: review_scope` (Step 3)
|
||||
is the knob; the receipt's `review_loop` records the outcome.
|
||||
|
||||
### Ship control flow
|
||||
|
||||
+2
-2
@@ -208,8 +208,8 @@ not a commit ID. Use `git diff <old-tree> <new-tree>` to compare these snapshots
|
||||
**Review policy** (docs/review-policy.md): only a finding that would fail a test,
|
||||
break production, or lose data or secrets blocks a pass; the rest is a PR-body
|
||||
note. Step 9's plan sets both reviews' depth (`--depth-floor standard|full` only
|
||||
raises it). Pass 1 reads the diff, later passes the range; the cap is two full
|
||||
passes and a fix-check. `NO FINDINGS` is complete. `ship: review_scope` (Step 3)
|
||||
raises it). Pass 1 reads the diff, later passes the range; the cap is two rounds,
|
||||
fix-checks included. `NO FINDINGS` is complete. `ship: review_scope` (Step 3)
|
||||
is the knob; the receipt's `review_loop` records the outcome.
|
||||
|
||||
### Ship control flow
|
||||
|
||||
+2
-2
@@ -210,8 +210,8 @@ not a commit ID. Use `git diff <old-tree> <new-tree>` to compare these snapshots
|
||||
**Review policy** (docs/review-policy.md): only a finding that would fail a test,
|
||||
break production, or lose data or secrets blocks a pass; the rest is a PR-body
|
||||
note. Step 9's plan sets both reviews' depth (`--depth-floor standard|full` only
|
||||
raises it). Pass 1 reads the diff, later passes the range; the cap is two full
|
||||
passes and a fix-check. `NO FINDINGS` is complete. `ship: review_scope` (Step 3)
|
||||
raises it). Pass 1 reads the diff, later passes the range; the cap is two rounds,
|
||||
fix-checks included. `NO FINDINGS` is complete. `ship: review_scope` (Step 3)
|
||||
is the knob; the receipt's `review_loop` records the outcome.
|
||||
|
||||
### Ship control flow
|
||||
|
||||
+9
-9
@@ -4,53 +4,53 @@
|
||||
"pins": {
|
||||
"agy/gstack-autoplan": 109666,
|
||||
"agy/gstack-plan-design-review": 109116,
|
||||
"agy/gstack-plan-devex-review": 114958,
|
||||
"agy/gstack-plan-devex-review": 114957,
|
||||
"agy/gstack-plan-eng-review": 143590,
|
||||
"agy/gstack-review": 104276,
|
||||
"codex/gstack-autoplan": 108409,
|
||||
"codex/gstack-plan-design-review": 110473,
|
||||
"codex/gstack-plan-devex-review": 116466,
|
||||
"codex/gstack-plan-devex-review": 116465,
|
||||
"codex/gstack-plan-eng-review": 144755,
|
||||
"codex/gstack-review": 103799,
|
||||
"copilot/gstack-autoplan": 109109,
|
||||
"copilot/gstack-plan-design-review": 109382,
|
||||
"copilot/gstack-plan-devex-review": 114420,
|
||||
"copilot/gstack-plan-devex-review": 114419,
|
||||
"copilot/gstack-plan-eng-review": 143755,
|
||||
"copilot/gstack-review": 103857,
|
||||
"cursor/gstack-autoplan": 108341,
|
||||
"cursor/gstack-plan-design-review": 107924,
|
||||
"cursor/gstack-plan-devex-review": 113690,
|
||||
"cursor/gstack-plan-devex-review": 113689,
|
||||
"cursor/gstack-plan-eng-review": 142345,
|
||||
"cursor/gstack-review": 122147,
|
||||
"factory/gstack-autoplan": 110619,
|
||||
"factory/gstack-plan-design-review": 111022,
|
||||
"factory/gstack-plan-devex-review": 117053,
|
||||
"factory/gstack-plan-devex-review": 117052,
|
||||
"factory/gstack-plan-eng-review": 145453,
|
||||
"factory/gstack-review": 125920,
|
||||
"gbrain/gstack-autoplan": 108748,
|
||||
"gbrain/gstack-plan-eng-review": 124972,
|
||||
"gemini/gstack-autoplan": 108899,
|
||||
"gemini/gstack-plan-design-review": 109210,
|
||||
"gemini/gstack-plan-devex-review": 114248,
|
||||
"gemini/gstack-plan-devex-review": 114247,
|
||||
"gemini/gstack-plan-eng-review": 143588,
|
||||
"gemini/gstack-review": 103679,
|
||||
"hermes/gstack-autoplan": 108751,
|
||||
"hermes/gstack-plan-eng-review": 125008,
|
||||
"kiro/gstack-autoplan": 108115,
|
||||
"kiro/gstack-plan-design-review": 107770,
|
||||
"kiro/gstack-plan-devex-review": 113542,
|
||||
"kiro/gstack-plan-devex-review": 113541,
|
||||
"kiro/gstack-plan-eng-review": 142207,
|
||||
"kiro/gstack-review": 121971,
|
||||
"openclaw/gstack-autoplan": 108571,
|
||||
"openclaw/gstack-plan-eng-review": 123487,
|
||||
"opencode/gstack-autoplan": 109090,
|
||||
"opencode/gstack-plan-design-review": 109238,
|
||||
"opencode/gstack-plan-devex-review": 114298,
|
||||
"opencode/gstack-plan-devex-review": 114297,
|
||||
"opencode/gstack-plan-eng-review": 143614,
|
||||
"opencode/gstack-review": 122769,
|
||||
"slate/gstack-autoplan": 108228,
|
||||
"slate/gstack-plan-design-review": 107847,
|
||||
"slate/gstack-plan-devex-review": 113616,
|
||||
"slate/gstack-plan-devex-review": 113615,
|
||||
"slate/gstack-plan-eng-review": 142276,
|
||||
"slate/gstack-review": 122059
|
||||
}
|
||||
|
||||
@@ -628,6 +628,24 @@ describe('gstack-developer-profile --check-mismatch', () => {
|
||||
// Error handling
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
describe('gstack-developer-profile --profile on an unparseable profile (#3763)', () => {
|
||||
test('exits non-zero, prints nothing to stdout, and names the file and the conflict markers', () => {
|
||||
fs.writeFileSync(path.join(tmpHome, 'developer-profile.json'), '{\n<<<<<<< HEAD\n "sessions": []\n=======\n "sessions": [1]\n>>>>>>> origin/main\n}\n');
|
||||
const r = runDev('--profile');
|
||||
expect(r.status).not.toBe(0);
|
||||
expect(r.stdout).toBe('');
|
||||
expect(r.stderr).toContain('developer-profile.json is not valid JSON');
|
||||
expect(r.stderr).toContain('git conflict markers');
|
||||
});
|
||||
|
||||
test('a valid profile still prints as before', () => {
|
||||
fs.writeFileSync(path.join(tmpHome, 'developer-profile.json'), '{"sessions":[]}\n');
|
||||
const r = runDev('--profile');
|
||||
expect(r.status).toBe(0);
|
||||
expect(r.stdout).toBe('{"sessions":[]}\n');
|
||||
});
|
||||
});
|
||||
|
||||
describe('gstack-developer-profile errors', () => {
|
||||
test('unknown subcommand exits non-zero', () => {
|
||||
const r = runDev('--not-a-real-subcommand');
|
||||
|
||||
@@ -136,6 +136,25 @@ describe('gstack-doctor', () => {
|
||||
expect(r.out).not.toContain('.brain-sync-status.json');
|
||||
});
|
||||
|
||||
test('artifacts sync: a synced JSON file with conflict markers warns by name (#3763)', () => {
|
||||
const f = makeFixture();
|
||||
syncOn(f, { status: 'ok', message: 'pushed 3 file(s)', drainable: 0, last_drain_at: iso(60), last_push_at: iso(60) });
|
||||
write(path.join(f.state, 'developer-profile.json'), '{\n<<<<<<< HEAD\n "v": 1\n=======\n "v": 2\n>>>>>>> origin/main\n}\n');
|
||||
const r = doctor(f);
|
||||
expect(r.row('artifacts sync').state).toBe('warn');
|
||||
expect(r.row('artifacts sync').detail).toBe('developer-profile.json holds git conflict markers and does not parse');
|
||||
write(path.join(f.state, 'developer-profile.json'), '{"v": 2}\n');
|
||||
expect(doctor(f).row('artifacts sync').state).toBe('ok');
|
||||
});
|
||||
|
||||
test('artifacts sync: a JSON hold is not attributed to the secret scanner (#3763)', () => {
|
||||
const f = makeFixture();
|
||||
syncOn(f, { status: 'held', message: 'held', held_count: 1, held: [{ path: 'developer-profile.json', rule: 'invalid-json', dependents: [], fixes: ['repair', 'skip'] }], drainable: 0, last_drain_at: iso(60), last_push_at: iso(60) });
|
||||
const r = doctor(f);
|
||||
expect(r.row('artifacts sync').state).toBe('warn');
|
||||
expect(r.row('artifacts sync').detail).toBe('status held (1 synced file(s) are not valid JSON; 1 held in all)');
|
||||
});
|
||||
|
||||
test('artifacts sync: held, then ok', () => {
|
||||
const f = makeFixture();
|
||||
syncOn(f, { status: 'held', message: 'held 2 file(s): sk-ant-secret', held_count: 2, held: [{ path: 'projects/p/notes.md', rule: 'anthropic_key', dependents: [], fixes: ['edit', 'skip'] }, { path: 'projects/p/b.md', rule: 'anthropic_key', dependents: [], fixes: ['edit', 'skip'] }], drainable: 0, last_drain_at: iso(60), last_push_at: iso(60) });
|
||||
|
||||
@@ -816,7 +816,7 @@ describe('ARTIFACTS_SYNC attention lines (sync status table)', () => {
|
||||
const held = runStart([], env);
|
||||
expect(held).toMatch(/^ARTIFACTS_SYNC: mode=full \| last_push=\S+ \| queue=\d+$/m);
|
||||
expect(attention(held)).toEqual([
|
||||
`ARTIFACTS_SYNC: attention: status=held held=1. The secret scan is holding back 1 file(s); everything else still syncs. See which files and how to fix them: ${path.join(ROOT, 'bin', 'gstack-brain-sync')} --status`,
|
||||
`ARTIFACTS_SYNC: attention: status=held held=1. Sync is holding back 1 file(s) that the secret scan flagged or that are not valid JSON; everything else still syncs. See which files and how to fix them: ${path.join(ROOT, 'bin', 'gstack-brain-sync')} --status`,
|
||||
]);
|
||||
expect(held).not.toContain('ghp_');
|
||||
expect(remoteFiles()).toContain('projects/p/ceo-plans/clean.md');
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
/**
|
||||
* Evidence for the /autoplan --ceo-only paid case (test/skill-e2e-autoplan-ceo-only.test.ts):
|
||||
* the run asked nothing, reviewed the plan through the CEO phase, skipped every
|
||||
* later phase, wrote ceo-review.md, ended status=reviewed, and wrote nothing in
|
||||
* HOME outside its declared roots. Pure over the transcript and files the test
|
||||
* collects, so test/autoplan-ceo-only.test.ts proves each check fails on a
|
||||
* synthetic negative control before any paid run.
|
||||
*/
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
|
||||
export const CEO_REVIEW_HEADINGS = [
|
||||
'## User challenges (direction kept)', '## Scope added', '## Conservative choices', '## Findings', '## Decision Audit Trail',
|
||||
] as const;
|
||||
|
||||
const QUESTION_TOOL = /(^|__)(AskUserQuestion|ExitPlanMode)$/;
|
||||
const PROSE_QUESTION = /\b(would you like|do you want|should i\b|want me to|shall i\b|which (?:option|mode|one) (?:do|would|should) you|reply with|let me know (?:if|which|whether))/i;
|
||||
const LATER_PHASE = /autoplan\/sections\/(?:design-phase|dx-phase|eng-phase|tasks-aggregator)\.md/;
|
||||
const DISPATCH_TOOL = /^(Agent|Task)$/;
|
||||
|
||||
export interface CeoOnlyEvidenceInput {
|
||||
transcript: any[];
|
||||
/** The session's final result text. */
|
||||
output: string;
|
||||
/** New files under HOME outside the declared roots (the test's before/after diff). */
|
||||
strayWrites: string[];
|
||||
/** Patterns the reviewed plan or ceo-review.md must contain for the seeded flaw. */
|
||||
mustMention: RegExp[];
|
||||
}
|
||||
export interface CeoOnlyEvidence { passed: boolean; reasons: string[]; runDir?: string; questions: string[] }
|
||||
|
||||
function blocks(transcript: any[]): Array<{ role: string; block: any }> {
|
||||
return transcript.flatMap(event => {
|
||||
const role = event?.type === 'assistant' ? 'assistant' : event?.type === 'user' ? 'user' : '';
|
||||
const content = event?.message?.content;
|
||||
return role && Array.isArray(content) ? content.map((block: any) => ({ role, block })) : [];
|
||||
});
|
||||
}
|
||||
|
||||
export function ceoOnlyEvidence(input: CeoOnlyEvidenceInput): CeoOnlyEvidence {
|
||||
const reasons: string[] = [];
|
||||
const all = blocks(input.transcript);
|
||||
const uses = all.filter(b => b.role === 'assistant' && b.block?.type === 'tool_use').map(b => b.block);
|
||||
const texts = all.filter(b => b.role === 'assistant' && b.block?.type === 'text').map(b => String(b.block.text ?? ''));
|
||||
const results = all.filter(b => b.role === 'user' && b.block?.type === 'tool_result')
|
||||
.map(b => (typeof b.block.content === 'string' ? b.block.content : JSON.stringify(b.block.content ?? '')));
|
||||
|
||||
const questions = [
|
||||
...uses.filter(u => QUESTION_TOOL.test(String(u.name))).map(u => `tool ${u.name}`),
|
||||
...[...texts, input.output].flatMap(t => t.split('\n').filter(line => line.trim().endsWith('?') && PROSE_QUESTION.test(line)).map(line => `prose: ${line.trim().slice(0, 160)}`)),
|
||||
];
|
||||
if (questions.length) reasons.push(`asked ${questions.length} question(s): ${questions.join(' | ')}`);
|
||||
|
||||
const handoff = uses.some(u => u.name === 'Bash' && /gstack-ceo-mode-handoff\b/.test(String(u.input?.command)) && /--ceo-only\b/.test(String(u.input?.command)))
|
||||
&& results.some(r => r.includes('(CEO-only run: Step 0E'));
|
||||
if (!handoff) reasons.push('no gstack-ceo-mode-handoff --ceo-only call with its CEO-only line');
|
||||
if (!uses.some(u => DISPATCH_TOOL.test(String(u.name)))) reasons.push('no CEO reviewer subagent was dispatched');
|
||||
const later = uses.map(u => String(u.name === 'Bash' ? u.input?.command : (u.input?.file_path ?? u.input?.path ?? u.input?.pattern ?? '')))
|
||||
.filter(arg => LATER_PHASE.test(arg));
|
||||
if (later.length) reasons.push(`loaded a skipped phase: ${later.map(arg => arg.slice(0, 160)).join(', ')}`);
|
||||
|
||||
const result = /^GSTACK_RESULT: skill=autoplan status=(\S+) run=(.+)$/m.exec(input.output.split('\n').filter(l => l.startsWith('GSTACK_RESULT: ')).pop() ?? '');
|
||||
if (!result) reasons.push('no GSTACK_RESULT line for autoplan in the final output');
|
||||
else if (result[1] !== 'reviewed') reasons.push(`GSTACK_RESULT status=${result[1]}, expected reviewed`);
|
||||
const runDir = result?.[2]?.trim();
|
||||
const reviewFile = runDir ? path.join(runDir, 'ceo-review.md') : '';
|
||||
const review = reviewFile && fs.existsSync(reviewFile) ? fs.readFileSync(reviewFile, 'utf8') : undefined;
|
||||
if (runDir && review === undefined) reasons.push(`${reviewFile} is missing`);
|
||||
if (review !== undefined) {
|
||||
const missing = CEO_REVIEW_HEADINGS.filter(h => !review.includes(h));
|
||||
if (missing.length) reasons.push(`ceo-review.md lacks ${missing.join(', ')}`);
|
||||
if (!/\|\s*\d+\s*\|/.test(review.slice(review.indexOf('## Decision Audit Trail')))) reasons.push('ceo-review.md has no Decision Audit Trail rows');
|
||||
const unmet = input.mustMention.filter(re => !re.test(review));
|
||||
if (unmet.length) reasons.push(`ceo-review.md never names the seeded flaw (${unmet.map(String).join(', ')})`);
|
||||
}
|
||||
if (input.strayWrites.length) {
|
||||
const dirs = [...new Set(input.strayWrites.map(f => f.split('/').slice(0, 2).join('/')))];
|
||||
reasons.push(`wrote ${input.strayWrites.length} file(s) outside its roots, under ${dirs.join(', ')}: ${input.strayWrites.slice(0, 5).join(', ')}`);
|
||||
}
|
||||
return { passed: reasons.length === 0, reasons, runDir, questions };
|
||||
}
|
||||
|
||||
/** Every file under root, relative, skipping the given relative prefixes. */
|
||||
export function listFiles(root: string, skip: string[] = []): string[] {
|
||||
const out: string[] = [];
|
||||
const walk = (dir: string) => {
|
||||
for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
|
||||
const abs = path.join(dir, entry.name);
|
||||
const rel = path.relative(root, abs);
|
||||
if (skip.some(s => rel === s || rel.startsWith(s + path.sep))) continue;
|
||||
if (entry.isDirectory()) walk(abs); else out.push(rel);
|
||||
}
|
||||
};
|
||||
if (fs.existsSync(root)) walk(root);
|
||||
return out.sort();
|
||||
}
|
||||
@@ -58,6 +58,16 @@ export function resultOf(out: string, phase: string, voice: 'native' | 'outside'
|
||||
}
|
||||
/** The sealed copy a completed voice is bound to. */
|
||||
export const boundOf = (out: string, phase: string, voice: 'native' | 'outside'): string => path.join(out, readRun(out).phases[phase][voice].output);
|
||||
/**
|
||||
* A `rec`/`all` reply that carries every open Critical/High finding item into the build (`<id>b`, no plan change),
|
||||
* so a test about another gate mechanism keeps its flow; `extra` tokens are appended to the except list.
|
||||
*/
|
||||
export function recCarrying(out: string, base = 'all', extra: string[] = []): string {
|
||||
const carry = (JSON.parse(fs.readFileSync(path.join(out, 'gate.json'), 'utf8')).items as Array<{ id: string; finding?: string[] }>).filter(i => i.finding).map(i => `${i.id}b`);
|
||||
const tokens = [...carry, ...extra];
|
||||
return tokens.length ? `${base} except ${tokens.join(' ')}` : base;
|
||||
}
|
||||
|
||||
export function attemptsOf(out: string, phase: string): { native: string; outside: string } {
|
||||
const run = readRun(out);
|
||||
return { native: run.phases[phase].native.attempt, outside: run.phases[phase].outside.attempt };
|
||||
|
||||
@@ -583,7 +583,7 @@ do not launch the downstream skill or open a browser.`,
|
||||
maxSkeletonBytes: 48_175, // GSTA-96 hoist (2026-10-26): lowered from 70_000 by the shared preamble that moved to sections/preamble.md, keeping the pre-hoist headroom of 4,888 bytes; measured 43,287. Before: Phase-specific outside coverage, native fallback, and harness guard.
|
||||
minUnionBytes: 85_000, // measured union 86,926
|
||||
mustContain: ['6 Decision Principles', 'TASTE DECISION', 'USER CHALLENGE', 'consensus', 'Restore Point'],
|
||||
maxSizeRatio: 1.1511, // Four validated outside invocations replace raw CLI calls; phases keep independent coverage. + Oct 7 wave B3: the Codex probe runs as a command (select-model capture lines, check-sandbox/probe-model/run-with-timeout subcommands) instead of being sourced; measured 114,297 / 101,979 = 1.1208 (2026-10-07). + Oct 7 wave CEO-28/DX-11: review prompts ask for a severity per finding and design proposals use the proposal gate; measured 114,533 / 101,979 = 1.1231 (2026-10-07). + Oct 7 wave guard (CEO-20): conditional `run_in_background` request plus the background-run recovery at each foreground dispatch site, and phase-close publishing each report with the `true autoplan-published <phase>` no-op before the next phase entry in a later message (CEO-1); measured 114,320 / 101,979 = 1.1210 (2026-10-07). Merged with the guard area: measured 115,149 / 101,979 = 1.1291 (2026-10-07). + Oct 8 wave (permission cards after a background reviewer): Step 1 puts RESTORE_PATH, and so every phase artifact, in the project's git-excluded `.gstack/tmp/autoplan/`, which Claude Code reads without a card after the skill turn ends; measured 115,270 / 101,979 = 1.1303 (2026-10-08). + fix wave PR D: the prerequisite offer is skipped in spawned/headless sessions (#1958); measured 115,267 / 101,979 = 1.1303 (2026-10-07). Both merged: measured 115,126 / 101,979 = 1.1289 (2026-10-08). + Honest Work wave merged (W2/W3 gate integrity in review/ship, W4 kernel + DONE line in every tier-2+ skill, W5/W7 in spec/ship): re-measured on the integrated tree (2026-10-10). + Oct 10 multi-agent wave B9/B10/B12: Phase 0 ESTIMATE and the phase-close timing record, the final gate as a gate list (`gstack-gate render/parse`, stable ids, gate_rev) and the URGENT block above the summary, offset by tightening the Phase 4 prose and the tasks-section; measured 114,980 / 101,979 = 1.1275 (2026-10-10). + Oct 10 multi-agent wave PR D1 (reality rows): the reality-rows check line in the CEO and Eng phase files, the Pre-Gate rule (unattended miss is `incomplete`, never a warning) and the two Pre-Gate cells; measured 117,042 / 101,979 = 1.1477 (2026-10-10). + GSTA-96 shared-preamble hoist (owner-approved raise, Garry 2026-10-25: "do it"): the union now holds sections/preamble.md, so a fresh invocation reads the pointer plus the shared file; measured 117,388 / 101,979 = 1.1511 (2026-10-25). Before: 1.149.
|
||||
maxSizeRatio: 1.1723, // GST-537 /autoplan --ceo-only (GStack Minder approved A1, 2026-10-31): sections/ceo-only.md (2,168 B) plus its Section index row, read only when the invocation names --ceo-only (test/autoplan-ceo-only.test.ts pins that a normal run never reads it); measured 119,545 / 101,979 = 1.1723 (2026-10-31), +2,276 B over main's 117,269 = 1.1499. Before: 1.1511. Four validated outside invocations replace raw CLI calls; phases keep independent coverage. + Oct 7 wave B3: the Codex probe runs as a command (select-model capture lines, check-sandbox/probe-model/run-with-timeout subcommands) instead of being sourced; measured 114,297 / 101,979 = 1.1208 (2026-10-07). + Oct 7 wave CEO-28/DX-11: review prompts ask for a severity per finding and design proposals use the proposal gate; measured 114,533 / 101,979 = 1.1231 (2026-10-07). + Oct 7 wave guard (CEO-20): conditional `run_in_background` request plus the background-run recovery at each foreground dispatch site, and phase-close publishing each report with the `true autoplan-published <phase>` no-op before the next phase entry in a later message (CEO-1); measured 114,320 / 101,979 = 1.1210 (2026-10-07). Merged with the guard area: measured 115,149 / 101,979 = 1.1291 (2026-10-07). + Oct 8 wave (permission cards after a background reviewer): Step 1 puts RESTORE_PATH, and so every phase artifact, in the project's git-excluded `.gstack/tmp/autoplan/`, which Claude Code reads without a card after the skill turn ends; measured 115,270 / 101,979 = 1.1303 (2026-10-08). + fix wave PR D: the prerequisite offer is skipped in spawned/headless sessions (#1958); measured 115,267 / 101,979 = 1.1303 (2026-10-07). Both merged: measured 115,126 / 101,979 = 1.1289 (2026-10-08). + Honest Work wave merged (W2/W3 gate integrity in review/ship, W4 kernel + DONE line in every tier-2+ skill, W5/W7 in spec/ship): re-measured on the integrated tree (2026-10-10). + Oct 10 multi-agent wave B9/B10/B12: Phase 0 ESTIMATE and the phase-close timing record, the final gate as a gate list (`gstack-gate render/parse`, stable ids, gate_rev) and the URGENT block above the summary, offset by tightening the Phase 4 prose and the tasks-section; measured 114,980 / 101,979 = 1.1275 (2026-10-10). + Oct 10 multi-agent wave PR D1 (reality rows): the reality-rows check line in the CEO and Eng phase files, the Pre-Gate rule (unattended miss is `incomplete`, never a warning) and the two Pre-Gate cells; measured 117,042 / 101,979 = 1.1477 (2026-10-10). + GSTA-96 shared-preamble hoist (owner-approved raise, Garry 2026-10-25: "do it"): the union now holds sections/preamble.md, so a fresh invocation reads the pointer plus the shared file; measured 117,388 / 101,979 = 1.1511 (2026-10-25). Before: 1.149.
|
||||
},
|
||||
spec: {
|
||||
skill: 'spec',
|
||||
|
||||
@@ -918,6 +918,12 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
'test/fixtures/autoplan-phase-handoff-6714.json','scripts/resolvers/learnings.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/preamble/generate-preamble-bash.ts', 'test/fixtures/plan-scope-recovery-av.json', 'test/fixtures/design-scope-checkpoint-at.json', 'scripts/resolvers/composition.ts', 'autoplan/**', 'lib/claude-journal-records.ts', 'lib/claude-owned-journal.ts', 'codex/**', 'bin/gstack-codex-probe', 'scripts/resolvers/review-dashboard.ts', 'scripts/resolvers/plan-gates.ts', 'scripts/resolvers/spec-review.ts', 'scripts/resolvers/outside-voice-steps.ts', 'scripts/resolvers/review-scope.ts', 'scripts/resolvers/outside-voice.ts', 'scripts/resolvers/design.ts', 'test/skill-e2e-autoplan-dual-voice.test.ts', 'bin/gstack-autoplan-snapshot.ts', 'test/fixtures/autoplan/t-ceo-omitted-obligations.json', 'test/fixtures/autoplan/u-ceo-original-loss.json', 'test/fixtures/autoplan/v-ceo-dangling-references.json',
|
||||
'scripts/resolvers/design-doc-discovery.ts', 'bin/gstack-design-doc-find', 'bin/gstack-ceo-plan-find', 'plan-ceo-review/**', 'ETHOS.md', 'plan-eng-review/**', 'plan-design-review/**', 'plan-devex-review/**', 'scripts/resolvers/testing.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/tasks-section.ts', 'test/helpers/autoplan-phase-observer.ts', 'scripts/resolve-codex-generation-model.ts', 'scripts/resolvers/constants.ts'],
|
||||
|
||||
// /autoplan --ceo-only: zero questions through the CEO phase (GST-537)
|
||||
'autoplan-ceo-only': ['autoplan/**', 'plan-ceo-review/**', 'ETHOS.md', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble-hoist.ts',
|
||||
'scripts/resolvers/preamble/generate-ask-user-format.ts', 'bin/gstack-state-root.sh', 'lib/state-root.ts',
|
||||
'lib/headless-artifacts.ts', 'bin/gstack-ceo-mode-handoff', 'test/helpers/autoplan-ceo-only-evidence.ts', 'test/helpers/hermetic-env.ts',
|
||||
'test/skill-e2e-autoplan-ceo-only.test.ts', 'test/fixtures/forcing-finding-seeds.ts'],
|
||||
|
||||
// Multi-provider benchmark adapters — live API smoke against real claude/codex/gemini CLIs
|
||||
'benchmark-providers-live': ['bin/gstack-model-benchmark', 'test/helpers/providers/**', 'test/helpers/benchmark-runner.ts', 'test/helpers/pricing.ts', 'test/skill-e2e-benchmark-providers.test.ts'],
|
||||
|
||||
@@ -1581,6 +1587,7 @@ export const E2E_TIERS: Record<string, 'gate' | 'periodic' | 'marathon'> = {
|
||||
|
||||
// Autoplan — periodic (not yet implemented)
|
||||
'autoplan-dual-voice': 'periodic',
|
||||
'autoplan-ceo-only': 'marathon', // Full /autoplan --ceo-only CEO review (349–984 s over 6 runs, GST-537); the gate (I5) and periodic lanes are at their caps
|
||||
|
||||
// Multi-provider benchmark — periodic (requires external CLIs + auth, paid)
|
||||
'benchmark-providers-live': 'periodic',
|
||||
@@ -1995,6 +2002,7 @@ export const E2E_KINDS: Record<string, 'rule' | 'behavior' | 'judge'> = {
|
||||
'benchmark-workflow': 'rule',
|
||||
'setup-deploy-workflow': 'rule',
|
||||
'autoplan-dual-voice': 'rule',
|
||||
'autoplan-ceo-only': 'rule',
|
||||
'benchmark-providers-live': 'rule',
|
||||
'scrape-match-path': 'behavior',
|
||||
'scrape-prototype-path': 'behavior',
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
/**
|
||||
* #3659: gstack-jsonl-merge read its inputs as UTF-8 but wrote them with
|
||||
* print(), so a redirected stdout on a legacy code page (cp1250 on the
|
||||
* reporter's Windows) raised UnicodeEncodeError on a Cyrillic line, and any
|
||||
* Windows Python rewrote LF as CRLF. The driver must write UTF-8 with LF
|
||||
* whatever the platform and code page. PYTHONIOENCODING=cp1252 reproduces the
|
||||
* legacy code page on every OS.
|
||||
*
|
||||
* Windows-safe by construction: the driver launches through
|
||||
* lib/script-invocation.ts (Git Bash on win32), files live under os.tmpdir().
|
||||
*/
|
||||
import { afterEach, beforeEach, expect, test } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { runScriptSync } from '../lib/script-invocation';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
let tmp = '';
|
||||
|
||||
beforeEach(() => { tmp = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-jsonl-enc-'))); });
|
||||
afterEach(() => fs.rmSync(tmp, { recursive: true, force: true }));
|
||||
|
||||
test('a line the stdout code page cannot encode merges as UTF-8 with LF endings (#3659)', () => {
|
||||
const base = '{"ts":"2026-01-01T00:00:00Z","note":"base"}\n';
|
||||
const ours = base + '{"ts":"2026-01-02T00:00:00Z","note":"ours"}\n';
|
||||
const theirsLine = '{"ts":"2026-01-03T00:00:00Z","note":"привет"}';
|
||||
const p = (n: string) => path.join(tmp, n);
|
||||
fs.writeFileSync(p('base'), base);
|
||||
fs.writeFileSync(p('ours'), ours);
|
||||
fs.writeFileSync(p('theirs'), base + theirsLine + '\n');
|
||||
const posix = (f: string) => f.replace(/\\/g, '/');
|
||||
const r = runScriptSync(path.join(ROOT, 'bin', 'gstack-jsonl-merge'), [posix(p('base')), posix(p('ours')), posix(p('theirs'))], {
|
||||
env: { ...process.env, PYTHONIOENCODING: 'cp1252', PYTHONUTF8: '0' }, encoding: 'utf8', timeout: 60_000,
|
||||
});
|
||||
expect(r.error).toBeUndefined();
|
||||
expect(r.status, r.stderr).toBe(0);
|
||||
const bytes = fs.readFileSync(p('ours'));
|
||||
expect(bytes.includes(0x0d)).toBe(false);
|
||||
expect(bytes.toString('utf8')).toBe(ours + theirsLine + '\n');
|
||||
});
|
||||
@@ -240,3 +240,46 @@ describe('api runner (supplied-input review)', () => {
|
||||
expect(nokey.stderr).toMatch(/OPENAI_API_KEY is not set.*\(OUTSIDE_RUNNER_UNAVAILABLE\)$/m);
|
||||
});
|
||||
});
|
||||
|
||||
describe('api runner output limits (GBRA-223 claim 6)', () => {
|
||||
let server: ReturnType<typeof Bun.serve>;
|
||||
const bodies: Array<{ url: string; body: any }> = [];
|
||||
beforeAll(() => {
|
||||
server = Bun.serve({
|
||||
port: 0,
|
||||
async fetch(req) {
|
||||
const url = new URL(req.url);
|
||||
const body = await req.json();
|
||||
bodies.push({ url: url.pathname, body });
|
||||
const cut = String(body.model).includes('cut');
|
||||
if (url.pathname === '/v1/messages') return Response.json({ model: body.model, stop_reason: cut ? 'max_tokens' : 'end_turn', content: [{ type: 'text', text: cut ? 'INPUT: ceo abc\n\nA long review cut off mid-sent' : REVIEW }], usage: { input_tokens: 10, output_tokens: body.max_tokens } });
|
||||
return Response.json({ model: body.model, choices: [{ finish_reason: cut ? 'length' : 'stop', message: { content: cut ? 'INPUT: ceo abc\n\nA long review cut off' : REVIEW } }], usage: { prompt_tokens: 10, completion_tokens: 50 } });
|
||||
},
|
||||
});
|
||||
});
|
||||
afterAll(() => server.stop(true));
|
||||
const anthropic = () => ({ ...cleanEnv(emptyBin), ANTHROPIC_API_KEY: 'ak-test', ANTHROPIC_BASE_URL: `http://127.0.0.1:${server.port}` });
|
||||
const openai = () => ({ ...cleanEnv(emptyBin), OPENAI_API_KEY: 'sk-test', OPENAI_BASE_URL: `http://127.0.0.1:${server.port}/v1` });
|
||||
const out = (name: string) => path.join(TMP, 'limits', name);
|
||||
|
||||
test('anthropic asks for 16384 output tokens by default and --max-tokens sets it; openai sends max_completion_tokens only when set', async () => {
|
||||
expect((await runAsync(['run', '--runner', 'api', '--prompt', promptFile, '--out', out('a1.md'), '--model', 'claude-opus-5-5'], anthropic())).status).toBe(0);
|
||||
expect((await runAsync(['run', '--runner', 'api', '--prompt', promptFile, '--out', out('a2.md'), '--model', 'claude-opus-5-5', '--max-tokens', '4000'], anthropic())).status).toBe(0);
|
||||
expect(bodies.filter(b => b.url === '/v1/messages').map(b => b.body.max_tokens)).toEqual([16384, 4000]);
|
||||
expect((await runAsync(['run', '--runner', 'api', '--prompt', promptFile, '--out', out('o1.md'), '--model', 'gpt-6-astra'], openai())).status).toBe(0);
|
||||
expect((await runAsync(['run', '--runner', 'api', '--prompt', promptFile, '--out', out('o2.md'), '--model', 'gpt-6-astra', '--max-tokens', '4000'], openai())).status).toBe(0);
|
||||
expect(bodies.filter(b => b.url === '/v1/chat/completions').map(b => b.body.max_completion_tokens ?? null)).toEqual([null, 4000]);
|
||||
});
|
||||
|
||||
test('a response stopped at the output limit is truncated (exit 1), never a completed or missing-markers review', async () => {
|
||||
for (const [model, env, tokens] of [['claude-cut', anthropic(), '16384'], ['gpt-cut', openai(), '50']] as const) {
|
||||
const file = out(`${model}.md`);
|
||||
const r = await runAsync(['run', '--runner', 'api', '--prompt', promptFile, '--out', file, '--model', model], env);
|
||||
expect(r.status).toBe(1);
|
||||
expect(r.stderr).toMatch(/api unavailable \(truncated\): .*output limit/);
|
||||
const meta = JSON.parse(fs.readFileSync(`${file}.meta.json`, 'utf8'));
|
||||
expect(meta).toMatchObject({ status: 'unavailable', reason: 'truncated' });
|
||||
expect(meta.detail).toContain(tokens);
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -205,6 +205,108 @@ describe('priced pilot', () => {
|
||||
expect(r.settled_unknown).toBe(1);
|
||||
expect(r.projection.verdict).toBe('unknown');
|
||||
});
|
||||
|
||||
test('#3653: each arm is projected on its own, so arms at different prices that fit the cap are not told they exceed it', () => {
|
||||
const d = tmp();
|
||||
const out = path.join(d, 'mixed');
|
||||
const s = startPilot({ out, cap: 2, n: 3, arms: ['gpt-6-astra', 'gpt-5.4-mini'], dryRun: passingDryRun(d) });
|
||||
expect(s.error).toBeUndefined();
|
||||
const usd: Record<string, number[]> = { 'gpt-6-astra': [0.001, 0.001, 0.001], 'gpt-5.4-mini': [0.10, 0.11, 0.12] };
|
||||
for (const a of s.record.attempts) settlePilot(out, a.attempt, usd[a.model]![a.row - 1]!);
|
||||
const r = pilotReport(out, { total: 10, workers: 1 });
|
||||
expect(r.projection.projected_usd).toBeCloseTo(1.11, 6);
|
||||
expect(r.projection.band_usd![0]).toBeCloseTo(1.11 - 4.303 * 0.01 / Math.sqrt(3) * 10, 4);
|
||||
expect(r.projection.band_usd![1]).toBeCloseTo(1.11 + 4.303 * 0.01 / Math.sqrt(3) * 10, 4);
|
||||
expect(r.projection.verdict).toBe('within');
|
||||
expect(r.projection.arms.map(a => [a.arm, a.n])).toEqual([['gpt-6-astra', 3], ['gpt-5.4-mini', 3]]);
|
||||
expect(renderPilotReport(r)).toContain('PILOT_ARM: arm=gpt-5.4-mini n=3 per_item=$0.1100');
|
||||
});
|
||||
|
||||
test('#3653: a priced arm mean is never multiplied across an arm settled unknown', () => {
|
||||
const d = tmp();
|
||||
const out = path.join(d, 'partial');
|
||||
const s = startPilot({ out, cap: 10, n: 3, arms: ['gpt-6-astra', 'gpt-5.4-mini'], dryRun: passingDryRun(d) });
|
||||
for (const a of s.record.attempts) settlePilot(out, a.attempt, a.model === 'gpt-6-astra' ? 0.0003 : 'unknown');
|
||||
const r = pilotReport(out, { total: 400 });
|
||||
expect(r.projection.projected_usd).toBeCloseTo(0.12, 6);
|
||||
expect(r.projection.verdict).toBe('unknown');
|
||||
expect(r.settled_unknown).toBe(3);
|
||||
const text = renderPilotReport(r);
|
||||
expect(text).toContain('known_arms=1/2');
|
||||
expect(text).toContain('--seat-metered');
|
||||
});
|
||||
|
||||
test('#3653: a declared seat-metered arm holds none of the cap, stays out of the verdict and gets its own line', () => {
|
||||
const d = tmp();
|
||||
const dry = passingDryRun(d);
|
||||
const est = startPilot({ out: path.join(d, 'probe'), cap: 100, n: 1, arms: ['gpt-6-astra', 'gpt-5.4-mini'], dryRun: dry }).record.attempts;
|
||||
const astra = est.find(a => a.model === 'gpt-6-astra')!.estimate_usd, mini = est.find(a => a.model === 'gpt-5.4-mini')!.estimate_usd;
|
||||
const cap = Number((3 * mini + 2 * astra).toFixed(4));
|
||||
expect(startPilot({ out: path.join(d, 'undeclared'), cap, n: 3, arms: ['gpt-5.4-mini', 'gpt-6-astra'], dryRun: dry }).error?.code).toBe('EVAL_BUDGET_EXHAUSTED');
|
||||
const out = path.join(d, 'seat');
|
||||
const s = startPilot({ out, cap, n: 3, arms: ['gpt-5.4-mini', 'gpt-6-astra'], seatMetered: ['gpt-6-astra'], dryRun: dry });
|
||||
expect(s.error).toBeUndefined();
|
||||
expect(s.record.seat_metered).toEqual(['gpt-6-astra']);
|
||||
expect(s.record.attempts.filter(a => a.model === 'gpt-6-astra').map(a => a.estimate_usd)).toEqual([0, 0, 0]);
|
||||
for (const a of s.record.attempts) settlePilot(out, a.attempt, a.model === 'gpt-6-astra' ? 'unknown' : 0.0003);
|
||||
const r = pilotReport(out, { total: 400, workers: 3 });
|
||||
expect(r.settled_unknown).toBe(0);
|
||||
expect(r.spend.summary.unknown_usd).toBe(0);
|
||||
expect(r.spend.max_estimate_usd).toBeCloseTo(mini, 6);
|
||||
expect(r.projection.projected_usd).toBeCloseTo(0.12, 6);
|
||||
expect(r.projection.verdict).toBe('within');
|
||||
expect(r.seat_metered).toEqual([{ arm: 'gpt-6-astra', calls: 3, settled: 3 }]);
|
||||
expect(renderPilotReport(r)).toContain('PILOT_SEAT_METERED: arm=gpt-6-astra calls=3 settled=3');
|
||||
});
|
||||
|
||||
test('#3653: a charge on a seat-metered attempt still counts as spent; the declaration names listed arms and is fixed for its --out', () => {
|
||||
const d = tmp();
|
||||
const dry = passingDryRun(d);
|
||||
const out = path.join(d, 'seat');
|
||||
const s = startPilot({ out, cap: 10, n: 1, arms: ['gpt-6-astra'], seatMetered: ['gpt-6-astra'], dryRun: dry });
|
||||
settlePilot(out, s.record.attempts[0]!.attempt, 0.05);
|
||||
const charged = pilotReport(out, { total: 1000 });
|
||||
expect(charged.spend.summary.spent_usd).toBeCloseTo(0.05, 6);
|
||||
expect(charged.seat_metered).toEqual([]);
|
||||
expect(charged.projection.arms.map(a => [a.arm, a.n])).toEqual([['gpt-6-astra', 1]]);
|
||||
expect(charged.projection.verdict).toBe('unknown');
|
||||
expect(() => startPilot({ out: path.join(d, 'stray'), cap: 10, n: 1, arms: ['gpt-6-astra'], seatMetered: ['gpt-5.4-mini'], dryRun: dry })).toThrow('--seat-metered');
|
||||
const switched = path.join(d, 'switched');
|
||||
startPilot({ out: switched, cap: 1, n: 3, arms: ['gpt-6-astra'], seatMetered: ['gpt-6-astra'], dryRun: dry });
|
||||
expect(() => startPilot({ out: switched, cap: 1, n: 3, arms: ['gpt-6-astra'], dryRun: dry })).toThrow('needs a new --out');
|
||||
expect(JSON.parse(fs.readFileSync(path.join(switched, 'pilot.json'), 'utf8')).seat_metered).toEqual(['gpt-6-astra']);
|
||||
});
|
||||
|
||||
test('#3653: an all-seat-metered pilot spends nothing and fits; one priced attempt is not enough for a verdict', () => {
|
||||
const d = tmp();
|
||||
const dry = passingDryRun(d);
|
||||
const seat = path.join(d, 'all-seat');
|
||||
const s = startPilot({ out: seat, cap: 1, n: 3, arms: ['gpt-6-astra'], seatMetered: ['gpt-6-astra'], dryRun: dry });
|
||||
for (const a of s.record.attempts) settlePilot(seat, a.attempt, 'unknown');
|
||||
const all = pilotReport(seat, { total: 50 });
|
||||
expect([all.projection.projected_usd, all.projection.verdict]).toEqual([0, 'within']);
|
||||
expect(renderPilotReport(all)).toContain('known_arms=0/0');
|
||||
const one = path.join(d, 'one');
|
||||
settlePilot(one, startPilot({ out: one, cap: 10, n: 1, arms: ['gpt-6-astra'], dryRun: dry }).record.attempts[0]!.attempt, 0.1);
|
||||
const r = pilotReport(one, { total: 50 });
|
||||
expect([r.projection.band_usd, r.projection.verdict]).toEqual([null, 'unknown']);
|
||||
expect(renderPilotReport(r)).toContain('settle at least 2 attempts per priced arm');
|
||||
});
|
||||
|
||||
test('#3653: a seat-metered arm still needs a price row, as the dry run does', () => {
|
||||
const d = tmp();
|
||||
const s = startPilot({ out: path.join(d, 'seat'), cap: 1, n: 2, arms: ['gpt-5.4-mini', 'seat-only-model'], seatMetered: ['seat-only-model'], dryRun: passingDryRun(d) });
|
||||
expect(s.error?.code).toBe('EVAL_PRICE_MISSING');
|
||||
});
|
||||
|
||||
test('#3653: pilot start --seat-metered reaches the ledger through the CLI', () => {
|
||||
const d = tmp();
|
||||
const out = path.join(d, 'cli');
|
||||
const r = spawnSync(BIN, ['pilot', 'start', '--out', out, '--cap', '10', '--arms', 'gpt-5.4-mini,gpt-6-astra', '--seat-metered', 'gpt-6-astra', '--dry-run', passingDryRun(d)], { cwd: d, encoding: 'utf8', timeout: 30_000 });
|
||||
expect(r.status).toBe(0);
|
||||
expect(r.stdout).toContain('PILOT_ATTEMPT: pilot-gpt-6-astra-1 model=gpt-6-astra estimate=$0.0000');
|
||||
expect(JSON.parse(fs.readFileSync(path.join(out, 'pilot.json'), 'utf8')).seat_metered).toEqual(['gpt-6-astra']);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
+46
-19
@@ -44,8 +44,23 @@ describe('nextStep: the transition table', () => {
|
||||
test('a fixed blocker earns full pass 2', () => expect(nextStep({ fullPasses: 1, last: 'full', fixedBlocking: true }).step).toBe('full'));
|
||||
test('below-floor fixes earn one fix-check', () => expect(nextStep({ fullPasses: 1, last: 'full', fixedBelow: true }).step).toBe('fix-check'));
|
||||
test('a fix touching gate code earns a full pass instead of a fix-check', () => expect(nextStep({ fullPasses: 1, last: 'full', fixedBelow: true, touchedGate: true }).step).toBe('full'));
|
||||
test('at the full-pass cap a fixed blocker gets a fix-check', () => expect(nextStep({ fullPasses: FULL_PASS_CAP, last: 'full', fixedBlocking: true }).step).toBe('fix-check'));
|
||||
test('at the cap a gate-touching fix still gets a fix-check, never a third full pass', () => expect(nextStep({ fullPasses: FULL_PASS_CAP, last: 'full', fixedBlocking: true, touchedGate: true }).step).toBe('fix-check'));
|
||||
// R51: two rounds is the cap, a fix-check included. A blocker open or fixed in round 2 stops for the owner (a grant may add
|
||||
// one scoped fix-check); below-floor fixes in round 2 finish with notes; a light pass whose fixes change no runtime lines is done.
|
||||
test('a blocker fixed in round 2 stops for the owner, never a third round', () => {
|
||||
expect(nextStep({ fullPasses: FULL_PASS_CAP, last: 'full', fixedBlocking: true })).toMatchObject({ step: 'stop', reason: expect.stringContaining('scoped fix-check') });
|
||||
expect(nextStep({ fullPasses: FULL_PASS_CAP, last: 'full', fixedBlocking: true, touchedGate: true }).step).toBe('stop');
|
||||
});
|
||||
test('below-floor fixes in round 2 finish with notes, a fix-check included', () => {
|
||||
expect(nextStep({ fullPasses: FULL_PASS_CAP, last: 'full', fixedBelow: true })).toMatchObject({ step: 'done', reason: expect.stringContaining('notes') });
|
||||
expect(nextStep({ fullPasses: 1, last: 'fix-check', fixedBelow: true })).toMatchObject({ step: 'done', reason: expect.stringContaining('notes') });
|
||||
expect(nextStep({ fullPasses: 1, last: 'fix-check', fixedBelow: true, fixedBlocking: true }).step).toBe('stop');
|
||||
});
|
||||
test('a light pass 1 whose below-floor fixes change no runtime lines is done; a runtime fix or a deeper diff keeps the fix-check', () => {
|
||||
expect(nextStep({ fullPasses: 1, last: 'full', fixedBelow: true, rangeRuntimeLines: 0 }, 'light').step).toBe('done');
|
||||
expect(nextStep({ fullPasses: 1, last: 'full', fixedBelow: true, rangeRuntimeLines: 3 }, 'light').step).toBe('fix-check');
|
||||
expect(nextStep({ fullPasses: 1, last: 'full', fixedBelow: true, rangeRuntimeLines: 0 }, 'full').step).toBe('fix-check');
|
||||
expect(nextStep({ fullPasses: 1, last: 'full', fixedBelow: true }, 'light').step).toBe('fix-check');
|
||||
});
|
||||
test('a blocker open at the cap stops', () => expect(nextStep({ fullPasses: FULL_PASS_CAP, last: 'full', blockingOpen: true }).step).toBe('stop'));
|
||||
test('a blocker open with no fix stops', () => expect(nextStep({ fullPasses: 1, last: 'full', blockingOpen: true }).step).toBe('stop'));
|
||||
test('a fix-check that finds a blocker stops, never a third full pass', () => {
|
||||
@@ -83,8 +98,12 @@ describe('planReview', () => {
|
||||
expect(p.depth).toBe('light');
|
||||
expect(roles(p)).toEqual([]);
|
||||
});
|
||||
test('code class pass 1: specialists at medium, the native adversarial at full, the outside voice at its own', () => {
|
||||
const p = planReview(base({}));
|
||||
test('non-risky code pass 1 is light at any size (R51): one batched reader carrying the Red Team prompt and the native adversarial', () => {
|
||||
const p = planReview(base({ diffLines: 900 }));
|
||||
expect([p.depth, roles(p)]).toEqual(['light', ['specialist:batched:full', 'adversarial-native:adversarial:full']]);
|
||||
});
|
||||
test('code class pass 1 at standard: specialists at medium, the native adversarial at full, the outside voice at its own', () => {
|
||||
const p = planReview(base({ floor: 'standard' }));
|
||||
expect(roles(p)).toEqual(['specialist:batched:medium', 'adversarial-native:adversarial:full', 'outside-voice:outside-voice:host']);
|
||||
expect(batch(p)).toEqual(['testing', 'security']);
|
||||
expect(p.skipped).toContain('maintainability, performance, simplification (lean ensemble: read only when forced)');
|
||||
@@ -162,10 +181,10 @@ describe('GSTA-131 pass-1 review repairs', () => {
|
||||
expect(scanNumbers({ 'test/fixtures/bench.json': ['{ "ratio": 0.74 }'] }).publishes).toBe(true);
|
||||
});
|
||||
test('--all-specialists forces every specialist; a code diff past 200 lines or with a CRITICAL adds the Red Team', () => {
|
||||
const all = planReview(base({ force: 'all' }));
|
||||
const all = planReview(base({ floor: 'standard', force: 'all' }));
|
||||
expect(all.dispatches.filter(d => d.role === 'specialist').length).toBe(1);
|
||||
expect([...batch(all)!].sort()).toEqual(['api-contract', 'data-migration', 'design', 'maintainability', 'performance', 'security', 'simplification', 'state-machines', 'testing']);
|
||||
const has = (o: Partial<PlanInput>) => roles(planReview(base(o))).some(r => r.startsWith('red-team'));
|
||||
const has = (o: Partial<PlanInput>) => roles(planReview(base({ floor: 'standard', ...o }))).some(r => r.startsWith('red-team'));
|
||||
expect(has({ diffLines: 120 })).toBe(false);
|
||||
expect(has({ diffLines: 201 })).toBe(true);
|
||||
expect(has({ diffLines: 120, criticalFound: true })).toBe(true);
|
||||
@@ -175,20 +194,20 @@ describe('GSTA-131 pass-1 review repairs', () => {
|
||||
describe('GSTA-131 coverage: Red Team and specialist arms', () => {
|
||||
const roles = (p: ReturnType<typeof planReview>) => p.dispatches.map(d => `${d.role}:${d.name}`);
|
||||
test('Red Team runs on code past 200 lines or after a CRITICAL, and on every gate diff', () => {
|
||||
expect(roles(planReview(base({ diffLines: 120 })))).not.toContain('red-team:red-team');
|
||||
expect(roles(planReview(base({ diffLines: 201 })))).toContain('red-team:red-team');
|
||||
expect(roles(planReview(base({ diffLines: 120, criticalFound: true })))).toContain('red-team:red-team');
|
||||
expect(roles(planReview(base({ diffLines: 120, floor: 'standard' })))).not.toContain('red-team:red-team');
|
||||
expect(roles(planReview(base({ diffLines: 201, floor: 'standard' })))).toContain('red-team:red-team');
|
||||
expect(roles(planReview(base({ diffLines: 120, floor: 'standard', criticalFound: true })))).toContain('red-team:red-team');
|
||||
expect(roles(planReview(base({ diffLines: 20, gateTagged: 1 })))).toContain('red-team:red-team');
|
||||
});
|
||||
test('scope arms select their specialist, state machines default on when scope is unknown, and --force all reaches docs class', () => {
|
||||
const picked = (scope: Record<string, string>) => batch(planReview(base({ scope: { SCOPE_STATE_MACHINE: 'false', ...scope } })));
|
||||
const picked = (scope: Record<string, string>) => batch(planReview(base({ floor: 'standard', scope: { SCOPE_STATE_MACHINE: 'false', ...scope } })));
|
||||
expect(picked({ SCOPE_MIGRATIONS: 'true' })).toContain('data-migration');
|
||||
expect(picked({ SCOPE_API: 'true' })).toContain('api-contract');
|
||||
expect(picked({ SCOPE_FRONTEND: 'true' })).toEqual(['testing', 'security', 'design']);
|
||||
expect(batch(planReview(base({ scope: {} })))).toContain('state-machines');
|
||||
expect(batch(planReview(base({ scope: { SCOPE_STATE_MACHINE: 'false' } })))).not.toContain('state-machines');
|
||||
expect(batch(planReview(base({ force: ['maintainability'] })))).toEqual(['testing', 'security', 'maintainability']);
|
||||
expect(batch(planReview(base({ gated: ['design'], scope: { SCOPE_FRONTEND: 'true', SCOPE_STATE_MACHINE: 'false' } })))).toEqual(['testing', 'security']);
|
||||
expect(batch(planReview(base({ floor: 'standard', force: ['maintainability'] })))).toEqual(['testing', 'security', 'maintainability']);
|
||||
expect(batch(planReview(base({ floor: 'standard', gated: ['design'], scope: { SCOPE_FRONTEND: 'true', SCOPE_STATE_MACHINE: 'false' } })))).toEqual(['testing', 'security']);
|
||||
const docsAll = planReview(base({ paths: ['docs/a.md'], force: 'all' }));
|
||||
expect(docsAll.class).toBe('docs');
|
||||
expect(docsAll.dispatches.filter(d => d.role === 'specialist').length).toBeGreaterThan(5);
|
||||
@@ -215,8 +234,12 @@ describe('review depth: the matrix', () => {
|
||||
expect(p.skipped.join('; ')).toContain('red-team (light depth');
|
||||
expect(planReview(mixed({ perPath: per({ 'src/app.ts': 48 }), adversarialRan: true, history: { fullPasses: 1, last: 'full', touchedGate: true, fixedBelow: true } })).dispatches.some(d => d.role === 'adversarial-native')).toBe(false);
|
||||
});
|
||||
test('runtime 51 is standard: thresholds read runtime lines, reviewers read their own hunks', () => {
|
||||
test('runtime 51 non-risky code stays light (R51: size never raises the depth)', () => {
|
||||
const p = planReview(mixed({ perPath: per({ 'src/app.ts': 51, 'test/a.test.ts': 400, 'README.md': 3 }), diffLines: 454 }));
|
||||
expect([p.depth, p.runtimeLines, roles(p)]).toEqual(['light', 51, ['specialist:batched:full', 'adversarial-native:adversarial:full']]);
|
||||
});
|
||||
test('standard (floor or raise): thresholds read runtime lines, reviewers read their own hunks', () => {
|
||||
const p = planReview(mixed({ perPath: per({ 'src/app.ts': 51, 'test/a.test.ts': 400, 'README.md': 3 }), diffLines: 454, floor: 'standard' }));
|
||||
expect([p.depth, p.runtimeLines]).toEqual(['standard', 51]);
|
||||
expect(roles(p)).toEqual(['specialist:batched:medium', 'adversarial-native:adversarial:full', 'outside-voice:outside-voice:host']);
|
||||
expect(batch(p)).toEqual(['testing', 'security']);
|
||||
@@ -274,14 +297,16 @@ describe('review depth: the matrix', () => {
|
||||
expect(depthFor({ cls: 'code', runtimeLines: 10, errors: [], floor: 'full' })).toBe('full');
|
||||
expect(depthFor({ cls: 'code', runtimeLines: 10, errors: [], floor: 'standard' })).toBe('standard');
|
||||
expect(depthFor({ cls: 'gate', runtimeLines: 10, errors: [], floor: 'standard' })).toBe('full');
|
||||
expect(depthFor({ cls: 'code', runtimeLines: 80, errors: [] })).toBe('standard');
|
||||
expect(depthFor({ cls: 'code', runtimeLines: 80, errors: [] })).toBe('light');
|
||||
expect(depthFor({ cls: 'code', runtimeLines: 3, errors: [], risk: ['data'] })).toBe('full');
|
||||
const p = planReview(mixed({ perPath: per({ 'src/app.ts': 10 }), floor: 'full' }));
|
||||
expect(p.depth).toBe('full');
|
||||
expect(p.depthReasons).toContain('floor:full');
|
||||
});
|
||||
test('depthOfCommitInput gives the same depth as the plan for the same input', () => {
|
||||
const i = { paths: ['src/app.ts', 'test/a.test.ts'], perPath: per({ 'src/app.ts': 51, 'test/a.test.ts': 9 }), gateTagged: 0 };
|
||||
expect(depthOfCommitInput(i)).toMatchObject({ depth: 'standard', runtimeLines: 51, rules: DEPTH_RULES, errors: [] });
|
||||
expect(depthOfCommitInput(i)).toMatchObject({ depth: 'light', runtimeLines: 51, rules: DEPTH_RULES, errors: [], risk: [] });
|
||||
expect(depthOfCommitInput({ ...i, added: { 'src/app.ts': ['const spend_cap_usd = 5;'] } })).toMatchObject({ depth: 'full', risk: ['money'] });
|
||||
expect(depthOfCommitInput({ ...i, gateTagged: null }).depth).toBe('full');
|
||||
});
|
||||
});
|
||||
@@ -300,6 +325,7 @@ describe('review depth: diff commands and the rules hash', () => {
|
||||
try {
|
||||
const src = fs.readFileSync(path.join(ROOT, 'lib/review-depth.ts'), 'utf8');
|
||||
fs.symlinkSync(path.join(ROOT, 'lib/ship-policy.ts'), path.join(dir, 'ship-policy.ts'));
|
||||
fs.symlinkSync(path.join(ROOT, 'lib/review-risk.ts'), path.join(dir, 'review-risk.ts'));
|
||||
const hashOf = (text: string) => {
|
||||
fs.writeFileSync(path.join(dir, 'review-depth.ts'), text);
|
||||
fs.writeFileSync(path.join(dir, 'print.ts'), "import { DEPTH_RULES } from './review-depth';\nconsole.log(DEPTH_RULES);\n");
|
||||
@@ -369,7 +395,7 @@ describe('D6: the scoped second pass after a blocking fix (GSTA-152 PR B)', () =
|
||||
expect(r(p)).toEqual(['fix-check:fix-check', 'red-team:red-team', 'adversarial-native:adversarial']);
|
||||
});
|
||||
test('the scoped pass counts toward the cap, and a blocker it leaves stops for the owner', () => {
|
||||
expect(nextStep(fixed(10, { fullPasses: FULL_PASS_CAP }), 'standard').step).toBe('fix-check');
|
||||
expect(nextStep(fixed(10, { fullPasses: FULL_PASS_CAP }), 'standard').step).toBe('stop');
|
||||
expect(nextStep({ fullPasses: FULL_PASS_CAP, last: 'full', blockingOpen: true, rangeRuntimeLines: 10 }, 'light').step).toBe('stop');
|
||||
const at = (m: number) => new Date(Date.UTC(2026, 9, 28, 10, m)).toISOString();
|
||||
const rows = [
|
||||
@@ -397,6 +423,7 @@ describe('D6: the scoped second pass after a blocking fix (GSTA-152 PR B)', () =
|
||||
const src = fs.readFileSync(path.join(ROOT, 'lib/review-depth.ts'), 'utf8');
|
||||
expect(src).toContain('SCOPED_RANGE_LINES = 150');
|
||||
fs.symlinkSync(path.join(ROOT, 'lib/ship-policy.ts'), path.join(dir, 'ship-policy.ts'));
|
||||
fs.symlinkSync(path.join(ROOT, 'lib/review-risk.ts'), path.join(dir, 'review-risk.ts'));
|
||||
fs.writeFileSync(path.join(dir, 'review-depth.ts'), src.replace('SCOPED_RANGE_LINES = 150', 'SCOPED_RANGE_LINES = 151'));
|
||||
fs.writeFileSync(path.join(dir, 'print.ts'), "import { DEPTH_RULES } from './review-depth';\nconsole.log(DEPTH_RULES);\n");
|
||||
const res = spawnSync(process.execPath, ['--no-env-file', '--config=/dev/null', path.join(dir, 'print.ts')], { encoding: 'utf8', timeout: 30_000 });
|
||||
@@ -491,11 +518,11 @@ describe('lean ensemble: one batched specialist at standard and full depth (GSTA
|
||||
expect(specialists(full).map(d => [d.name, d.effort, d.checklists])).toEqual([['batched', 'medium', ['testing', 'security', 'state-machines']]]);
|
||||
});
|
||||
test('a standard code plan reads one batched reader on its own hunks, and a forced checklist joins it', () => {
|
||||
const p = planReview(base({ diffLines: 400, scope: { SCOPE_BACKEND: 'true', SCOPE_STATE_MACHINE: 'false' } }));
|
||||
const p = planReview(base({ diffLines: 400, floor: 'standard', scope: { SCOPE_BACKEND: 'true', SCOPE_STATE_MACHINE: 'false' } }));
|
||||
expect(specialists(p).map(d => [d.name, d.checklists])).toEqual([['batched', ['testing', 'security']]]);
|
||||
expect(roles(p)).toContain('red-team:red-team:full');
|
||||
expect(batch(planReview(base({ force: ['performance', 'simplification'] })))).toEqual(['testing', 'security', 'performance', 'simplification']);
|
||||
expect(specialists(planReview(base({ tiers: false })))[0]!.effort).toBe('host');
|
||||
expect(batch(planReview(base({ floor: 'standard', force: ['performance', 'simplification'] })))).toEqual(['testing', 'security', 'performance', 'simplification']);
|
||||
expect(specialists(planReview(base({ floor: 'standard', tiers: false })))[0]!.effort).toBe('host');
|
||||
});
|
||||
});
|
||||
|
||||
@@ -56,6 +56,7 @@ const NO_LOSS_REPLAYS = {
|
||||
'G1': "GSTA-167 variant G Arm G1: live no-loss replay of both prose voices with part 1's prompts, 2 passes, normalizer v2, paired rule against pooled C' (parts 1 and 3; specialists at medium as in C and C2)",
|
||||
'L': "GSTA-167 variant G Arm L: live no-loss replay of one batched reader (testing and security plus scoped checklists) and the Red Team on its findings, 2 passes, scored L∘G1 against G1 on the paired rule (part 2)",
|
||||
'GST-257': "GST-257 PR 1 dispatch-superset no-loss check (test/review-targeted.test.ts): on every fixture diff the branch dispatches main's reviewers at 8be7f21 unchanged plus only the targeted reader, and over the last 100 non-release commits on main it lost none; adds the targeted role and brief to the G1 and L bytes, whose existing reviewer text is unchanged",
|
||||
'R51': "R51 risk-tier routing no-loss re-score (GSTA-167): the saved panel findings (349 records over 24 of the 26 no-loss units) re-scored against the branch's plan lose no blocker whose only finders are roles it drops, and every gate, security, user-data and money fixture keeps main's reviewers plus the full panel (test/review-targeted.test.ts, test/review-risk-routing.test.ts); reviewer text is unchanged",
|
||||
} as const;
|
||||
type Replay = keyof typeof NO_LOSS_REPLAYS;
|
||||
const REVIEWER_BRIEFS: Record<string, { sha: string; replay: Replay }> = {
|
||||
@@ -77,7 +78,8 @@ const REVIEWER_BRIEFS: Record<string, { sha: string; replay: Replay }> = {
|
||||
'review/specialists/testing.md': { sha: 'd6cc2bef7be562387d0c0a22fefe646dff9c295f1d34d8453a6cb496584150e4', replay: '#3638' },
|
||||
'review/specialists/targeted.md': { sha: '7f92a0d0a8319c3c0e076fb3b1a4689c45120b2be2c14dfc7d4cb03aafa9d6a5', replay: 'GST-257' },
|
||||
'scripts/resolvers/fix-check.ts': { sha: '712bf5b0c439e96bac32f19cbe25394dabb4c94aa66600100bfde9b100e15a08', replay: '#3638' },
|
||||
'lib/review-depth.ts': { sha: '4542785a1e217d8dd054985978b42c055b9fd79082579cb7664f187f7066e48e', replay: 'GST-257' },
|
||||
'lib/review-depth.ts': { sha: 'd649569a163c03476c6e0c5ea295f5d02088f96aab85fbc7d558f2ac2b27006a', replay: 'R51' },
|
||||
'lib/review-risk.ts': { sha: 'dbef7475d467e17340b21a2bacb52a16a7ccb1ee66291900ed2d36dfb5cfb7f2', replay: 'R51' },
|
||||
};
|
||||
|
||||
describe('reviewer briefs stay at their no-loss-tested bytes (GSTA-167)', () => {
|
||||
|
||||
@@ -522,7 +522,7 @@ describe('Capy pass program (fake host, real bins)', () => {
|
||||
fs.mkdirSync(files, { recursive: true });
|
||||
const fc: Record<string, string> = {};
|
||||
for (const [k, v] of [['fix_list', '1. src/a.ts: guard v80\n'], ['hunks', 'diff --git a/src/a.ts b/src/a.ts\n+export const v81 = 81;\n'], ['tests', 'bun test: 3 pass, 0 fail\n'], ['callers', 'none\n']]) { fc[k] = path.join(files, k + '.txt'); fs.writeFileSync(fc[k]!, v!); }
|
||||
const planFlags = ['--pass', '2', '--last', 'full', '--fixed-blocking'];
|
||||
const planFlags = ['--pass', '1', '--last', 'full', '--fixed-below'];
|
||||
const missing = await runPass(host(dir, env), { base: 'main', out: path.join(dir, '.state/run/fc0'), pass: 'x19fc0', planFlags });
|
||||
expect(missing).toContain('STOP: gstack-review-dispatch pass exited');
|
||||
expect(missing).toContain('the plan dispatches a fix-check: pass needs --fix-list, --hunks and --tests');
|
||||
|
||||
@@ -35,18 +35,18 @@ describe('gstack-review-plan', () => {
|
||||
});
|
||||
test('--json carries the class, the next step, every dispatch with its effort and the audit effort; --no-tiers renders specialists and audits at host effort', () => {
|
||||
const r = repo({ 'src/a.ts': Array.from({ length: 80 }, (_, i) => `export const v${i} = ${i};`).join('\n') + '\n' });
|
||||
const plan = JSON.parse(run(r, ['--json']).stdout);
|
||||
const plan = JSON.parse(run(r, ['--json', '--depth-floor', 'standard']).stdout);
|
||||
expect(plan).toMatchObject({ class: 'code', next: { step: 'full' }, pass: 0 });
|
||||
expect(plan.dispatches.filter((d: any) => d.role === 'specialist').map((d: any) => d.effort)).toEqual(expect.arrayContaining(['medium']));
|
||||
expect(plan.dispatches.filter((d: any) => d.role === 'specialist').every((d: any) => d.effort === 'medium')).toBe(true);
|
||||
expect(plan.dispatches.find((d: any) => d.role === 'adversarial-native').effort).toBe('full');
|
||||
expect(plan.audit_effort).toBe('light');
|
||||
expect(run(r, []).stdout).toContain('DISPATCH: specialist batched effort=medium checklists=testing,security,state-machines');
|
||||
const host = JSON.parse(run(r, ['--json', '--no-tiers']).stdout);
|
||||
expect(run(r, ['--depth-floor', 'standard']).stdout).toContain('DISPATCH: specialist batched effort=medium checklists=testing,security,state-machines');
|
||||
const host = JSON.parse(run(r, ['--json', '--no-tiers', '--depth-floor', 'standard']).stdout);
|
||||
expect(host.audit_effort).toBe('host');
|
||||
expect(host.dispatches.filter((d: any) => d.role === 'specialist').every((d: any) => d.effort === 'host')).toBe(true);
|
||||
expect(host.dispatches.find((d: any) => d.role === 'adversarial-native').effort).toBe('full');
|
||||
expect(run(r, ['--no-tiers']).stdout).toContain('DISPATCH: specialist batched effort=host checklists=testing,security,state-machines');
|
||||
expect(run(r, ['--no-tiers', '--depth-floor', 'standard']).stdout).toContain('DISPATCH: specialist batched effort=host checklists=testing,security,state-machines');
|
||||
});
|
||||
test('a below-floor fix after pass 1 plans one fix-check', () => {
|
||||
const r = run(repo({ 'src/a.ts': 'export const a = 1;\n' }), ['--pass', '1', '--last', 'full', '--fixed-below']);
|
||||
@@ -132,7 +132,7 @@ describe('gstack-review-plan: review depth', () => {
|
||||
const r = repo({ 'src/a.ts': code(10), 'test/a.test.ts': code(200) });
|
||||
const out = run(r, []).stdout;
|
||||
const head = spawnSync('git', ['rev-parse', 'HEAD'], { cwd: r.dir, encoding: 'utf8', timeout: 10_000 }).stdout.trim();
|
||||
expect(out).toMatch(/^REVIEW_DEPTH: light runtime_lines=10 rules=[0-9a-f]{16} reasons=runtime=10<=50 codes=none$/m);
|
||||
expect(out).toMatch(/^REVIEW_DEPTH: light runtime_lines=10 rules=[0-9a-f]{16} reasons=runtime=10 codes=none$/m);
|
||||
expect(out).toContain(`GATE_SCAN: tagged=0 head=${head}`);
|
||||
expect(out).toContain('DISPATCH: specialist batched effort=full checklists=testing,security,state-machines,red-team');
|
||||
expect(out).toContain('DISPATCH: adversarial-native adversarial effort=full reason=backtest inconclusive (GSTA-147 D5)');
|
||||
@@ -149,7 +149,7 @@ describe('gstack-review-plan: review depth', () => {
|
||||
});
|
||||
test('--json carries depth, runtime_lines, rules, errors, gate_scan and every dispatch\'s exclude array', () => {
|
||||
const r = repo({ 'src/a.ts': code(80), 'README.md': '# y\n' });
|
||||
const plan = JSON.parse(run(r, ['--json']).stdout);
|
||||
const plan = JSON.parse(run(r, ['--json', '--depth-floor', 'standard']).stdout);
|
||||
expect(plan).toMatchObject({ depth: 'standard', step_depth: 'standard', runtime_lines: 80, errors: [], codes: [], gate_scan: { tagged: 0 } });
|
||||
expect(plan.rules).toMatch(/^[0-9a-f]{16}$/);
|
||||
expect(plan.dispatches.find((d: any) => d.name === 'batched').exclude).toEqual([':(exclude)README.md']);
|
||||
@@ -157,7 +157,7 @@ describe('gstack-review-plan: review depth', () => {
|
||||
});
|
||||
test('--depth-floor full raises a light diff, never lowers a full one, and an invalid floor is a usage error', () => {
|
||||
const r = repo({ 'src/a.ts': code(5) });
|
||||
expect(run(r, ['--depth-floor', 'full']).stdout).toMatch(/^REVIEW_DEPTH: full runtime_lines=5 .*reasons=runtime=5<=50,floor:full /m);
|
||||
expect(run(r, ['--depth-floor', 'full']).stdout).toMatch(/^REVIEW_DEPTH: full runtime_lines=5 .*reasons=runtime=5,floor:full /m);
|
||||
const gate = repo({ 'src/a.ts': code(5), '.github/workflows/ci.yml': 'on: push\n' });
|
||||
expect(run(gate, ['--depth-floor', 'standard']).stdout).toMatch(/^REVIEW_DEPTH: full /m);
|
||||
expect(run(r, ['--depth-floor', 'light']).status).toBe(2);
|
||||
@@ -190,7 +190,7 @@ describe('gstack-review-plan: the scoped second pass (GSTA-152 D6)', () => {
|
||||
const out = run(r, [...pass2, '--range-base', rangeBase]).stdout;
|
||||
expect(out).toMatch(/^REVIEW_PLAN: class=code pass=2 next=scoped dispatches=3 /m);
|
||||
expect(out).toContain(`FIX_RANGE: runtime_lines=24 base=${rangeBase}`);
|
||||
expect(out).toMatch(/^NEXT: scoped \(a blocking finding was fixed on a standard diff with 24 runtime lines in the fix range/m);
|
||||
expect(out).toMatch(/^NEXT: scoped \(a blocking finding was fixed on a light diff with 24 runtime lines in the fix range/m);
|
||||
expect(out).toContain('DISPATCH: fix-check fix-check effort=full');
|
||||
expect(out).toContain('DISPATCH: red-team red-team effort=full');
|
||||
expect(out).toContain('DISPATCH: adversarial-native adversarial effort=full');
|
||||
@@ -221,7 +221,7 @@ describe('gstack-review-plan: the scoped second pass (GSTA-152 D6)', () => {
|
||||
fs.writeFileSync(path.join(r.dir, 'src', 'b.ts'), code(2, 'b'));
|
||||
git('add', '.'); git('commit', '-qm', 'fix: revert the workflow edit');
|
||||
const out = run(r, [...pass2, '--range-base', rangeBase]).stdout;
|
||||
expect(out).toMatch(/^REVIEW_DEPTH: standard /m);
|
||||
expect(out).toMatch(/^REVIEW_DEPTH: light .* step=standard$/m);
|
||||
expect(out).toMatch(/^NEXT: full \(a blocking finding was fixed and a fix touched a gate-class path/m);
|
||||
expect(JSON.parse(run(r, [...pass2, '--range-base', rangeBase, '--json']).stdout)).toMatchObject({ next: { step: 'full' }, fix_range: { touched_gate: true } });
|
||||
}, 30_000);
|
||||
|
||||
@@ -0,0 +1,123 @@
|
||||
/**
|
||||
* R51: review scales to risk, not size (docs/review-policy.md#review-depth, #risk-classes). The risk classes
|
||||
* (lib/review-risk.ts), the round cap, and the routing table: every gate,
|
||||
* security, user-data and money fixture still plans the full panel, and non-risky code is light at any size.
|
||||
*/
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { classifyDiff, nextStep, planReview, scanNumbers, type PlanInput } from '../lib/review-depth';
|
||||
import { matchRisk, RISK_RULES } from '../lib/review-risk';
|
||||
import { targetedClasses } from '../lib/review-targeted';
|
||||
import { parseReviewLoop, receiptSummaryLine } from '../lib/ship-receipt';
|
||||
|
||||
const ROOT = path.join(import.meta.dir, '..');
|
||||
const risk = (codePaths: string[], added: Record<string, string[]> = {}, scopeAuth = false) => [...matchRisk({ codePaths, added, removed: {}, scopeAuth }).keys()];
|
||||
|
||||
describe('risk classes', () => {
|
||||
// Value: protects=security and user-data code reaches the full panel; fails_when=a trust-boundary or data-loss signal is missed,
|
||||
// or an unrelated path or line raises a class; seam=matchRisk
|
||||
test('security and data: paths and lines, each with a negative', () => {
|
||||
expect(risk(['src/core/trust/owner-actions.ts'])).toEqual(['security']);
|
||||
expect(risk(['lib/redact-patterns.ts'])).toEqual(['security']);
|
||||
expect(risk(['src/run.ts'], { 'src/run.ts': ['const child = spawn(cmd, { shell: true });'] })).toEqual(['security']);
|
||||
expect(risk(['src/view.tsx'], { 'src/view.tsx': ['<div dangerouslySetInnerHTML={{ __html: body }} />'] })).toEqual(['security']);
|
||||
expect(risk(['bin/x.sh'], { 'bin/x.sh': ['eval "$CMD"'] })).toEqual(['security']);
|
||||
expect(risk(['src/run.ts'], { 'src/run.ts': ['const m = re.exec(line);', 'const s = trustee.name;'] })).toEqual([]);
|
||||
expect(risk(['src/core/persistence/git-door.ts'])).toEqual(['data']);
|
||||
expect(risk(['src/db/users.ts'], { 'src/db/users.ts': ['await sql`DELETE FROM users WHERE id = ${id}`;'] })).toEqual(['data']);
|
||||
expect(risk(['app/models/user.rb'], { 'app/models/user.rb': ['User.where(stale: true).destroy_all'] })).toEqual(['data']);
|
||||
expect(risk(['src/list.ts'], { 'src/list.ts': ['// delete from the list when done', 'arr.splice(i, 1);'] })).toEqual([]);
|
||||
for (const line of ['db.exec("delete from users");', 'await sql`delete from sessions where id = 1`;', 'DELETE FROM users;', 'drop database app;', 'DROP SCHEMA s;',
|
||||
'await sql`DELETE FROM ${table} WHERE id = ?`;', 'DELETE FROM users u WHERE u.stale', 'DELETE FROM sessions USING users WHERE x', 'DELETE FROM users RETURNING id', 'const q = "DELETE FROM " + table;'])
|
||||
expect(risk(['src/db.ts'], { 'src/db.ts': [line] })).toEqual(['data']);
|
||||
for (const line of ["const r = spawn('sh', ['-c', cmd], { timeout: 1 });", "Bun.spawn(['bash', '-lc', x]);", "fs.rmSync(t, { recursive: true, force: true });", 'rm -rf "$DIR"'])
|
||||
expect(risk(['src/run.ts'], { 'src/run.ts': [line] })).toEqual(['security']);
|
||||
expect(risk(['scripts/test-pr-profile.ts', 'lib/storyboard.ts'])).toEqual([]);
|
||||
expect(RISK_RULES).toMatch(/^[0-9a-f]{16}$/);
|
||||
});
|
||||
// Value: protects=the targeted pass keeps its exact classes now that its matchers live in lib/review-risk.ts; fails_when=the move
|
||||
// changes a money, auth or secret verdict; seam=targetedClasses over matchRisk
|
||||
test('money, auth and secret come from the same matchers the targeted pass reads', () => {
|
||||
const lines = { 'src/run.ts': ['const spend_cap_usd = 5;', 'const k = process.env.OPENAI_API_KEY;'] };
|
||||
expect(risk(['src/run.ts', 'src/auth/session.ts'], lines, true)).toEqual(['money', 'auth', 'secret']);
|
||||
expect(targetedClasses({ paths: ['src/run.ts', 'src/auth/session.ts'], added: lines, removed: {}, scope: { SCOPE_AUTH: 'true' }, gateTagged: 0 }).classes).toEqual(['money', 'auth', 'secret']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('UI text and SVGs stay code (R51: the docs paths are deferred to an allowlist classifier)', () => {
|
||||
// Value: protects=a copy edit, a Vue binding or a namespaced SVG script never plans as docs; fails_when=any of them leaves the code
|
||||
// class or its lines skip the risk scan; seam=classifyDiff
|
||||
test.each([
|
||||
['a JSX copy edit', 'src/App.tsx', ['<h1>Old copy</h1>'], ['<h1>New copy</h1>']],
|
||||
['a Vue binding', 'src/App.vue', ['<div :title="user.name">Profile</div>'], ['<div :title="leak(document.cookie)">Profile</div>']],
|
||||
['a locale value', 'locales/en.yml', ['save: "Save"'], ['save: "<img src=x onerror=alert(1)>"']],
|
||||
['a namespaced SVG script', 'docs/chart.svg', [], ['<svg xmlns:s="http://www.w3.org/2000/svg"><s:script>window.x=42</s:script></svg>']],
|
||||
])('%s is code', (_n, p, removed, added) => {
|
||||
expect(classifyDiff({ paths: [p], gateTagged: 0, added: { [p]: added }, removed: { [p]: removed }, perPath: [{ path: p, lines: removed.length + added.length, binary: false }] }).class).toBe('code');
|
||||
});
|
||||
});
|
||||
|
||||
describe('round 2 never ships an unread gate edit', () => {
|
||||
// Value: protects=a gate-class edit after round 1 always gets the owner's eyes; fails_when=a round-2 or fix-check gate edit
|
||||
// returns done; seam=nextStep
|
||||
test('a gate edit in round 2 or in the fix-check stops for the owner; a plain below-floor fix there finishes', () => {
|
||||
expect(nextStep({ fullPasses: 2, last: 'full', fixedBelow: true, touchedGate: true })).toMatchObject({ step: 'stop', reason: expect.stringContaining('gate-class') });
|
||||
expect(nextStep({ fullPasses: 1, last: 'fix-check', touchedGate: true })).toMatchObject({ step: 'stop', reason: expect.stringContaining('gate-class') });
|
||||
expect(nextStep({ fullPasses: 2, last: 'full', fixedBelow: true }).step).toBe('done');
|
||||
});
|
||||
});
|
||||
|
||||
const none = scanNumbers({});
|
||||
const plan = (o: Partial<PlanInput>) => planReview({ paths: ['src/app.ts'], scope: { SCOPE_STATE_MACHINE: 'false' }, gateTagged: 0, diffLines: 10, numbers: none, history: { fullPasses: 0, last: 'none' }, ...o });
|
||||
const roles = (p: ReturnType<typeof planReview>) => [...new Set(p.dispatches.map(d => d.role))];
|
||||
const FULL_PANEL = ['specialist', 'red-team', 'adversarial-native', 'outside-voice'];
|
||||
const captured = (name: string): Partial<PlanInput> => {
|
||||
const x = JSON.parse(fs.readFileSync(path.join(ROOT, 'test/fixtures/review-depth', name + '.json'), 'utf8'));
|
||||
return { paths: x.paths, deleted: x.deleted, stampOnly: x.stamp_only ?? [], scope: x.scope, gateTagged: x.gate_tagged, diffLines: x.diff_lines, numbers: x.numbers, perPath: x.per_path };
|
||||
};
|
||||
const one = (p: string, lines: string[] = ['export const a = 1;']) => ({ paths: [p], added: { [p]: lines }, removed: {}, perPath: [{ path: p, lines: lines.length, binary: false }] });
|
||||
|
||||
describe('routing: every gate, security, user-data and money fixture plans the full panel', () => {
|
||||
// Value: protects=R51 never lowers review on risky code; fails_when=a gate or risk fixture plans less than outside voice, Red Team,
|
||||
// batched specialist and the native adversarial pass, or loses the targeted reader it had; seam=planReview
|
||||
test.each([
|
||||
['captured gate fixture', captured('code'), []],
|
||||
['gate path', one('.github/workflows/ci.yml', ['on: push']), []],
|
||||
['gate tag', { ...one('src/a.ts'), gateTagged: 2 }, ['gate']],
|
||||
['deleted test', { ...one('src/a.ts'), deleted: ['test/a.test.ts'] }, []],
|
||||
['auth scope', { ...one('src/auth/session.ts'), scope: { SCOPE_AUTH: 'true' } }, ['auth']],
|
||||
['migration scope', { ...one('db/migrate/001.sql', ['ALTER TABLE users ADD COLUMN email text;']), scope: { SCOPE_MIGRATIONS: 'true' } }, []],
|
||||
['secret path', one('lib/credentials.ts'), ['secret']],
|
||||
['secret line', one('src/client.ts', ['const k = process.env.OPENAI_API_KEY;']), ['secret']],
|
||||
['money path', one('src/billing/invoice.ts'), ['money']],
|
||||
['money line', one('src/charge.ts', ['export const spend_cap_usd = 5;']), ['money']],
|
||||
['security path', one('lib/redact-patterns.ts'), []],
|
||||
['security line', one('src/run.ts', ['const run = new Function(body);']), []],
|
||||
['user-data path', one('src/core/persistence/store.ts'), []],
|
||||
['user-data line', one('src/db/purge.ts', ['await db.users.deleteMany({ where: { stale: true } });']), []],
|
||||
] as const)('%s', (_n, o, targeted) => {
|
||||
const p = plan({ ...(o as Partial<PlanInput>), targeted: { classes: [...targeted] } });
|
||||
expect(p.depth).toBe('full');
|
||||
expect(FULL_PANEL.filter(r => !roles(p).includes(r as never))).toEqual([]);
|
||||
expect(roles(p).includes('targeted')).toBe(targeted.length > 0);
|
||||
});
|
||||
test('non-risky code is light at any size: one batched reader with the Red Team prompt and the native adversarial', () => {
|
||||
const p = plan({ ...one('src/util.ts', Array.from({ length: 900 }, (_, n) => `export const v${n} = ${n};`)), diffLines: 900 });
|
||||
expect([p.depth, roles(p)]).toEqual(['light', ['specialist', 'adversarial-native']]);
|
||||
const docs = plan({ paths: ['docs/evals/run.md'], perPath: [{ path: 'docs/evals/run.md', lines: 400, binary: false }], numbers: scanNumbers({ 'docs/evals/run.md': ['recall 33/41'] }) });
|
||||
expect([docs.depth, roles(docs)]).toEqual(['light', ['outside-voice']]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('the receipt records the tier and the rounds', () => {
|
||||
// Value: protects=a receipt carries review_loop.rounds and the summary line prints depth and rounds; fails_when=the parser drops
|
||||
// rounds or a legacy loop without it gains one; seam=parseReviewLoop and receiptSummaryLine
|
||||
test('rounds parse and print; a legacy loop prints no rounds', () => {
|
||||
expect(parseReviewLoop({ passes: 1, blocking_found: 0, deferred: 0, fix_checks: 1, rounds: 2 })).toMatchObject({ rounds: 2 });
|
||||
const r = { pr: 7, head: 'a'.repeat(40), base: 'main', review_depth: { depth: 'light' }, review_loop: { passes: 1, blocking_found: 0, deferred: 0, fix_checks: 1, rounds: 2 } } as never;
|
||||
expect(receiptSummaryLine(r)).toContain('depth=light passes=1 blocking_found=0 deferred=0 fix_checks=1 rounds=2');
|
||||
const legacy = { pr: 7, head: 'a'.repeat(40), base: 'main', review_loop: { passes: 2, blocking_found: 1, deferred: 0 } } as never;
|
||||
expect(receiptSummaryLine(legacy)).not.toContain('rounds=');
|
||||
});
|
||||
});
|
||||
@@ -30,7 +30,7 @@ describe('late defers from Step 11', () => {
|
||||
{ skill: 'adversarial-review', via: 'ship', timestamp: '2026-10-26T10:25:00Z', findings: [{ fingerprint: 'd:1:w', blocking: true, action: 'deferred' }] },
|
||||
];
|
||||
const s = summarizeReviewLoop(rows, '2026-10-26T09:30:00Z');
|
||||
expect(s).toEqual({ passes: 2, blocking_found: 3, deferred: 2, noted: 0, fixed: 1, scope: 'frozen' });
|
||||
expect(s).toEqual({ passes: 2, blocking_found: 3, deferred: 2, noted: 0, fixed: 1, scope: 'frozen', rounds: 2 });
|
||||
expect(reviewLoopLine(s)).toBe('REVIEW_LOOP: passes=2 blocking_found=3 deferred=2 noted=0 fixed=1 scope=frozen');
|
||||
expect(LATE_DEFER_SKILL).toBe('review-late-defer');
|
||||
// A `review` row logged without a start token would be the newest review row and read UNVERIFIED; the late-defer row is not a diff review at all.
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
/**
|
||||
* #3541: `gstack-review-scope summary` reads the review log through
|
||||
* gstack-review-read, a bash script. Spawned by path it never starts on
|
||||
* Windows (no shebangs), so summary always refused with an empty
|
||||
* "gstack-review-read failed:" and /ship could not write its review-loop file.
|
||||
*
|
||||
* Windows-safe by construction: both bins launch through
|
||||
* lib/script-invocation.ts (Git Bash on win32), state lives under os.tmpdir().
|
||||
*/
|
||||
import { afterEach, beforeEach, expect, test } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { runScriptSync } from '../lib/script-invocation';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const SLUG = 'review-scope-launch';
|
||||
|
||||
let tmp = '', repo = '', state = '', env: NodeJS.ProcessEnv = {};
|
||||
|
||||
beforeEach(() => {
|
||||
tmp = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-rs-launch-')));
|
||||
repo = path.join(tmp, 'repo');
|
||||
state = path.join(tmp, 'state');
|
||||
fs.mkdirSync(repo);
|
||||
fs.mkdirSync(path.join(state, 'projects', SLUG), { recursive: true });
|
||||
const gitconfig = path.join(tmp, 'gitconfig');
|
||||
fs.writeFileSync(gitconfig, '');
|
||||
env = {
|
||||
...process.env, HOME: tmp, USERPROFILE: tmp, GSTACK_STATE_ROOT: state, GSTACK_HOME: state, GSTACK_PROJECT_SLUG: SLUG,
|
||||
GIT_CONFIG_GLOBAL: gitconfig, GIT_CONFIG_NOSYSTEM: '1', GIT_DIR: undefined, GIT_WORK_TREE: undefined, GIT_INDEX_FILE: undefined,
|
||||
};
|
||||
const git = (...args: string[]) => {
|
||||
const r = spawnSync('git', args, { cwd: repo, env, encoding: 'utf8', timeout: 30_000 });
|
||||
expect(r.status, r.stderr).toBe(0);
|
||||
};
|
||||
git('init', '-q', '-b', 'feat');
|
||||
fs.writeFileSync(path.join(repo, 'a.txt'), 'a\n');
|
||||
git('add', 'a.txt');
|
||||
git('-c', 'user.email=fixture@example.test', '-c', 'user.name=Fixture', '-c', 'commit.gpgsign=false', 'commit', '-qm', 'fixture');
|
||||
});
|
||||
|
||||
afterEach(() => fs.rmSync(tmp, { recursive: true, force: true }));
|
||||
|
||||
test('summary reads the review log through gstack-review-read (#3541)', () => {
|
||||
const rows = [
|
||||
{ skill: 'review', via: 'ship', timestamp: '2026-10-26T10:00:00Z', review_scope: 'frozen', findings: [{ fingerprint: 'a:1:x', blocking: true, action: 'fixed' }] },
|
||||
{ skill: 'review', via: 'ship', timestamp: '2026-10-26T10:30:00Z', review_scope: 'frozen', findings: [] },
|
||||
];
|
||||
fs.writeFileSync(path.join(state, 'projects', SLUG, 'feat-reviews.jsonl'), rows.map(r => JSON.stringify(r)).join('\n') + '\n');
|
||||
const out = path.join(tmp, 'review-loop.json');
|
||||
const r = runScriptSync(path.join(ROOT, 'bin', 'gstack-review-scope'), ['summary', '--since', '2026-10-26T09:00:00Z', '--out', out, '--json'], {
|
||||
cwd: repo, env, encoding: 'utf8', timeout: 120_000,
|
||||
});
|
||||
expect(r.error).toBeUndefined();
|
||||
expect(r.status, r.stderr).toBe(0);
|
||||
const summary = JSON.parse(fs.readFileSync(out, 'utf8'));
|
||||
expect(summary).toMatchObject({ passes: 2, blocking_found: 1, fixed: 1, scope: 'frozen' });
|
||||
}, 150_000);
|
||||
@@ -208,7 +208,7 @@ describe('loop summary', () => {
|
||||
{ skill: 'review', timestamp: '2026-10-26T10:40:00Z', findings: [{ fingerprint: 'e:1:v', blocking: true }] },
|
||||
];
|
||||
const s = summarizeReviewLoop(rows, '2026-10-26T09:30:00Z');
|
||||
expect(s).toEqual({ passes: 2, blocking_found: 2, deferred: 1, noted: 1, fixed: 1, scope: 'frozen' });
|
||||
expect(s).toEqual({ passes: 2, blocking_found: 2, deferred: 1, noted: 1, fixed: 1, scope: 'frozen', rounds: 2 });
|
||||
expect(reviewLoopLine(s)).toBe('REVIEW_LOOP: passes=2 blocking_found=2 deferred=1 noted=1 fixed=1 scope=frozen');
|
||||
});
|
||||
|
||||
@@ -223,7 +223,7 @@ describe('loop summary', () => {
|
||||
{ skill: 'review', via: 'ship', review_binding: { state: 'changed', started_at: '2026-10-26T09:00:00.000Z' }, findings: [{ fingerprint: 'old:1:q', action: 'noted' }] },
|
||||
{ skill: 'review', via: 'ship', findings: [{ fingerprint: 'undated:1:q', action: 'noted' }] },
|
||||
];
|
||||
expect(summarizeReviewLoop(rows, '2026-10-26T09:30:00Z')).toEqual({ passes: 3, blocking_found: 1, deferred: 0, noted: 2, fixed: 1, scope: 'frozen' });
|
||||
expect(summarizeReviewLoop(rows, '2026-10-26T09:30:00Z')).toEqual({ passes: 3, blocking_found: 1, deferred: 0, noted: 2, fixed: 1, scope: 'frozen', rounds: 3 });
|
||||
});
|
||||
});
|
||||
|
||||
@@ -421,7 +421,7 @@ describe('gstack-review-scope (bin)', () => {
|
||||
{ skill: 'review', timestamp: '2026-10-26T10:40:00Z', findings: [{ fingerprint: 'e:1:v', blocking: true, action: 'fixed' }] },
|
||||
];
|
||||
fs.writeFileSync(path.join(logDir, `${branch}-reviews.jsonl`), rows.map(r => JSON.stringify(r)).join('\n') + '\n{not json\n');
|
||||
const expected = { passes: 2, blocking_found: 2, deferred: 1, noted: 1, fixed: 1, scope: 'frozen' };
|
||||
const expected = { passes: 2, blocking_found: 2, deferred: 1, noted: 1, fixed: 1, scope: 'frozen', rounds: 2 };
|
||||
const out = path.join(repo.root, 'review-loop.json');
|
||||
const r = runBin(repo, 'gstack-review-scope', ['summary', '--since', '2026-10-26T09:30:00Z', '--out', out, '--json']);
|
||||
expect(r.status).toBe(EXIT.ok);
|
||||
|
||||
@@ -110,8 +110,8 @@ describe('planReview dispatches the targeted reader', () => {
|
||||
test('at light, standard and full depth when a class hit, never when none did', () => {
|
||||
expect(plan({ targeted: { classes: ['money'] } })).toMatchObject({ depth: 'light' });
|
||||
expect(roles(plan({ targeted: { classes: ['money'] } }))).toContain('targeted full');
|
||||
expect(plan({ diffLines: 80, perPath: [{ path: 'src/a.ts', lines: 80, binary: false }], targeted: { classes: ['secret'] } }).depth).toBe('standard');
|
||||
expect(roles(plan({ diffLines: 80, perPath: [{ path: 'src/a.ts', lines: 80, binary: false }], targeted: { classes: ['secret'] } }))).toContain('targeted full');
|
||||
expect(plan({ diffLines: 80, perPath: [{ path: 'src/a.ts', lines: 80, binary: false }], targeted: { classes: ['secret'] }, floor: 'standard' }).depth).toBe('standard');
|
||||
expect(roles(plan({ diffLines: 80, perPath: [{ path: 'src/a.ts', lines: 80, binary: false }], targeted: { classes: ['secret'] }, floor: 'standard' }))).toContain('targeted full');
|
||||
expect(plan({ gateTagged: 1, targeted: { classes: ['gate'] } }).depth).toBe('full');
|
||||
expect(roles(plan({ gateTagged: 1, targeted: { classes: ['gate'] } }))).toContain('targeted full');
|
||||
expect(roles(plan({ targeted: { classes: [] } }))).not.toContain('targeted full');
|
||||
@@ -183,6 +183,18 @@ function planRepo(f: Fixture) {
|
||||
const run = (args: string[]) => spawnSync(PLAN, ['--base', 'main', ...(args.includes('--gate-tagged') ? [] : ['--gate-tagged', '0']), ...args], { cwd: dir, encoding: 'utf8', timeout: 60_000, env: { ...process.env, GSTACK_STATE_ROOT: state } });
|
||||
return { dir, run, write };
|
||||
}
|
||||
/**
|
||||
* R51 (risk, not size): the depth each fixture plans now. Every risk or gate fixture plans full and keeps every role main
|
||||
* dispatched, plus the full panel; a non-risky fixture keeps main's reviewers except the roles listed (size no longer raises it).
|
||||
*/
|
||||
const FULL_PANEL = ['specialist', 'red-team', 'adversarial-native', 'outside-voice'];
|
||||
const R51: Record<string, { depth: string; panel: boolean; dropped?: string[] }> = {
|
||||
'none-light': { depth: 'light', panel: false }, 'none-standard': { depth: 'light', panel: false, dropped: ['outside-voice'] }, 'none-docs': { depth: 'light', panel: false },
|
||||
'none-tests': { depth: 'light', panel: false }, 'none-costume': { depth: 'light', panel: false }, 'none-gate-path': { depth: 'full', panel: true },
|
||||
'secret-light': { depth: 'full', panel: true }, 'secret-path': { depth: 'full', panel: true }, 'secret-removed': { depth: 'full', panel: true }, 'gate-tagged': { depth: 'full', panel: true },
|
||||
'money-light': { depth: 'full', panel: true }, 'money-path': { depth: 'full', panel: true }, 'money-removed': { depth: 'full', panel: true }, 'money-standard': { depth: 'full', panel: true },
|
||||
'auth-path': { depth: 'full', panel: true }, 'money-fix-check': { depth: 'full', panel: false }, 'money-pass2': { depth: 'full', panel: false }, 'gate-repeat': { depth: 'full', panel: false },
|
||||
};
|
||||
const field = (out: string, prefix: string, key: string) => new RegExp(`^${prefix}.* ${key}=(\\S+)`, 'm').exec(out)?.[1];
|
||||
|
||||
describe('gstack-review-plan prints TARGETED and dispatches the targeted reader', () => {
|
||||
@@ -190,17 +202,23 @@ describe('gstack-review-plan prints TARGETED and dispatches the targeted reader'
|
||||
// standard and full depth and on fix-check, scoped and repeat steps, is still dispatched unchanged, and the only addition is the
|
||||
// targeted reader on a class diff; fails_when=the change drops, renames or re-efforts any reviewer, or adds the reader on a diff
|
||||
// with no class; seam=bin/gstack-review-plan end to end; no_claim=that the reader finds the defect (the paid replay's question)
|
||||
test('the dispatch-superset no-loss check: on every fixture the branch dispatches main\'s reviewers plus only the targeted reader', () => {
|
||||
// R51 changes the table: risk and gate fixtures plan full (every role main dispatched, plus the full panel), and the one
|
||||
// non-risky standard fixture drops only the outside voice; fails_when=a risk fixture loses a role main dispatched or misses
|
||||
// the full panel, or a non-risky fixture loses a role the R51 table does not list
|
||||
test('the dispatch-superset no-loss check: risk and gate fixtures keep every role main dispatched plus the full panel; non-risky ones drop only what R51 lists', () => {
|
||||
expect(Object.keys(MAIN).sort()).toEqual(Object.keys(FIXTURES).sort());
|
||||
expect(new Set(Object.values(MAIN).map(m => m.depth))).toEqual(new Set(['light', 'standard', 'full']));
|
||||
expect(Object.keys(R51).sort()).toEqual(Object.keys(FIXTURES).sort());
|
||||
const role = (d: string) => d.split(' ')[0]!;
|
||||
for (const [name, f] of Object.entries(FIXTURES)) {
|
||||
const r = planRepo(f).run(f.args ?? []);
|
||||
expect(r.status, `${name}: ${r.stderr}`).toBe(0);
|
||||
const got = r.stdout.split('\n').filter(l => l.startsWith('DISPATCH: ')).map(l => l.slice('DISPATCH: '.length));
|
||||
const want = MAIN[name]!;
|
||||
expect(r.stdout, name).toContain(`REVIEW_DEPTH: ${want.depth} `);
|
||||
expect(got.filter(d => !want.main.includes(d)), name).toEqual(want.classes === 'none' ? [] : ['targeted targeted effort=full']);
|
||||
expect(want.main.filter(d => !got.includes(d)), name).toEqual([]);
|
||||
const want = MAIN[name]!, now = R51[name]!;
|
||||
expect(r.stdout, name).toContain(`REVIEW_DEPTH: ${now.depth} `);
|
||||
const roles = new Set(got.map(role));
|
||||
expect([...new Set(want.main.map(role))].filter(x => !roles.has(x)), name).toEqual(now.dropped ?? []);
|
||||
if (now.panel) expect(FULL_PANEL.filter(x => !roles.has(x)), name).toEqual([]);
|
||||
expect(roles.has('targeted'), name).toBe(want.classes !== 'none');
|
||||
expect(field(r.stdout, 'TARGETED:', 'classes'), name).toBe(want.classes);
|
||||
}
|
||||
}, 180_000);
|
||||
|
||||
@@ -18,7 +18,7 @@ import * as path from 'node:path';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
|
||||
const SPAWN_BIN_BY_PATH = /\b(?:spawn|spawnSync|execFile|execFileSync)\(\s*(?:path\.)?join\(\s*(?:BIN_DIR|binDir|BIN|bin|binRoot|BIN_ROOT)\b/;
|
||||
const SPAWN_BIN_BY_PATH = /\b(?:spawn|spawnSync|execFile|execFileSync)\(\s*(?:path\.)?join\(\s*(?:(?:BIN_DIR|binDir|BIN|bin|binRoot|BIN_ROOT)\b|(?:path\.)?dirname\(|import\.meta\.dir\b)/;
|
||||
const SPAWN_BARE_BASH = /\b(?:spawn|spawnSync|execFile|execFileSync|execSync)\(\s*['"]bash['"]|Bun\.spawn(?:Sync)?\(\s*\[\s*['"]bash['"]|\[\s*['"]bash['"]\s*,\s*['"]-l?c['"]/;
|
||||
const COMMENT = /^\s*(?:\/\/|\*|\/\*)/;
|
||||
/** `const X = path.resolve(…, 'bin', 'gstack-…')`: a bin path held in a constant, then spawned by name (#3190's gstack-config). */
|
||||
@@ -34,6 +34,8 @@ const KNOWN_SITES: Record<string, string> = {
|
||||
'bin/gstack-issue-links': 'outside this wave: the gstack-issue-guard spawn moves to runScriptSync',
|
||||
// Found when the bin-constant rule was added (fix wave PR 5, item 10b).
|
||||
'bin/gstack-brain-cache': 'outside this wave: the gstack-config salience read moves to runScriptSync',
|
||||
// Found when the scanner learned the import.meta.url bin path (#3541).
|
||||
'bin/gstack-ship-receipt': 'the gstack-session-kind spawn moves to runScriptSync in the #3541 ship-receipt follow-up',
|
||||
'lib/autoplan-admission.ts': 'returns before the spawn on win32; the bin/gstack-lease touch moves to runScriptSync with that bin\'s owner',
|
||||
};
|
||||
|
||||
@@ -88,6 +90,8 @@ describe('#3197: shebang bins and bash go through lib/script-invocation.ts', ()
|
||||
test('the scanner catches both shapes', () => {
|
||||
expect(SPAWN_BIN_BY_PATH.test("spawnSync(path.join(BIN_DIR, 'gstack-wtree'), [], { timeout: 1 })")).toBe(true);
|
||||
expect(SPAWN_BIN_BY_PATH.test('spawnSync(join(BIN_DIR, "gstack-slug"), { timeout: 1 })')).toBe(true);
|
||||
expect(SPAWN_BIN_BY_PATH.test("spawnSync(path.join(path.dirname(Bun.fileURLToPath(import.meta.url)), 'gstack-review-read'), [], { timeout: 1 })")).toBe(true);
|
||||
expect(SPAWN_BIN_BY_PATH.test("spawnSync(join(import.meta.dir, 'gstack-slug'), [], { timeout: 1 })")).toBe(true);
|
||||
expect(SPAWN_BARE_BASH.test("spawnSync('bash', ['-c', cmd], { timeout: 1 })")).toBe(true);
|
||||
expect(SPAWN_BARE_BASH.test('const argv = ["bash", "-c", command];')).toBe(true);
|
||||
expect(SPAWN_BARE_BASH.test("spawnSync(process.execPath, [script], { timeout: 1 })")).toBe(false);
|
||||
|
||||
@@ -86,7 +86,7 @@ posixTest('a scripted actor that fixes the diff, defers the pre-existing defect
|
||||
await act(f);
|
||||
const verdict = await shipFrozenScopeFailures(f, { exitReason: 'success' });
|
||||
expect(verdict.failures).toEqual([]);
|
||||
expect(verdict.evidence.loop).toEqual({ passes: 2, blocking_found: 2, deferred: 1, noted: 0, fixed: 1, scope: 'frozen' });
|
||||
expect(verdict.evidence.loop).toEqual({ passes: 2, blocking_found: 2, deferred: 1, noted: 0, fixed: 1, rounds: 2, scope: 'frozen' });
|
||||
const rows = verdict.evidence.rows as Array<Record<string, any>>;
|
||||
expect(rows.map(r => r.pass)).toEqual([1, 2]);
|
||||
expect(rows[0]!.findings.map((x: any) => [x.fingerprint, x.action])).toEqual([[FROZEN_SCOPE_FIX_IDENTITY, 'auto-fixed'], [FROZEN_SCOPE_DEFER_IDENTITY, 'deferred']]);
|
||||
|
||||
@@ -6,6 +6,7 @@
|
||||
import { afterEach, describe, expect, setDefaultTimeout, test } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { laneConcurrency } from '../lib/ship-bin-common';
|
||||
import { lineOf, PREFLIGHT, shipFixture, type ShipFx } from './helpers/ship-bin-fixture';
|
||||
|
||||
setDefaultTimeout(60_000);
|
||||
@@ -107,6 +108,47 @@ describe('gstack-ship-preflight', () => {
|
||||
expect(lineOf(zero.stdout, 'LANE')).toMatch(/^LANE: tests ZERO-RUN exit=0 tests_ran=0/);
|
||||
});
|
||||
|
||||
// Value: protects=Step 5 can run its lanes one at a time, and each lane keeps its own tests_ran (GST-851: GSTA-2 saw six lanes start within
|
||||
// 60 ms on a 4-core machine on gstack-evals#115); fails_when=--lanes serial lanes overlap, a lane loses its count, or a bad --lanes value runs;
|
||||
// why_new=runLanes started every lane at once with no serial mode; seam=runLanes; no_claim=how long a lane takes
|
||||
test('--lanes serial runs one lane at a time and keeps each lane\'s count; a bad --lanes value is usage', () => {
|
||||
const f = fx();
|
||||
const log = path.join(f.root, 'lane-order.log');
|
||||
const lane = (n: string) => `${n}=echo start ${n} >> '${log}'; sleep 1; echo end ${n} >> '${log}'; printf " 2 pass\\n 0 fail\\nRan 2 tests across 1 file. [5.00ms]\\n"`;
|
||||
const r = f.run(PREFLIGHT, ['--tests-only', '--base', 'main', '--lanes', 'serial', '--lane', lane('a'), '--lane', lane('b')]);
|
||||
expect(r.status).toBe(0);
|
||||
const order = fs.readFileSync(log, 'utf8').trim().split('\n');
|
||||
expect(order.length).toBe(4);
|
||||
expect(order.map(l => l.split(' ')[0])).toEqual(['start', 'end', 'start', 'end']);
|
||||
expect(order[1]!.split(' ')[1]).toBe(order[0]!.split(' ')[1]);
|
||||
expect(r.stdout.split('\n').filter(l => l.startsWith('LANE: ')).map(l => l.replace(/ log=\S+$/, ''))).toEqual(['LANE: a pass exit=0 tests_ran=2', 'LANE: b pass exit=0 tests_ran=2']);
|
||||
expect(lineOf(r.stdout, 'LANES')).toBe('LANES: serial (--lanes)');
|
||||
expect(f.run(PREFLIGHT, ['--tests-only', '--base', 'main', '--lanes', 'fast', '--lane', LANE]).status).toBe(2);
|
||||
expect(f.run(PREFLIGHT, ['--tests-only', '--base', 'main', '--lane', LANE, '--lanes']).status).toBe(2);
|
||||
});
|
||||
|
||||
// Value: protects=parallel lanes report in the order given even when a later lane finishes first, and an unflagged run names the core count
|
||||
// behind its default (GST-851); fails_when=results come back in finish order or the LANES line drops its reason; why_new=runLanes became a
|
||||
// worker pool; seam=runLanes, LANES line; no_claim=which default any given machine picks
|
||||
test('--lanes parallel keeps lane order when a later lane finishes first; unflagged LANES names the core count', () => {
|
||||
const f = fx();
|
||||
const lane = (n: string, wait: number) => `${n}=sleep ${wait}; printf " 1 pass\\n 0 fail\\nRan 1 tests across 1 file. [5.00ms]\\n"`;
|
||||
const r = f.run(PREFLIGHT, ['--tests-only', '--base', 'main', '--lanes', 'parallel', '--lane', lane('slow', 1), '--lane', lane('fast', 0)]);
|
||||
expect(r.status).toBe(0);
|
||||
expect(lineOf(r.stdout, 'LANES')).toBe('LANES: parallel (--lanes)');
|
||||
expect(r.stdout.split('\n').filter(l => l.startsWith('LANE: ')).map(l => l.split(' ')[1])).toEqual(['slow', 'fast']);
|
||||
expect(lineOf(f.run(PREFLIGHT, ['--tests-only', '--base', 'main', '--lane', LANE]).stdout, 'LANES')).toMatch(/^LANES: (serial|parallel) \(\d+ cores\)$/);
|
||||
});
|
||||
|
||||
// Value: protects=a small machine runs lanes one at a time without a flag, a large one keeps running them together, and the flag overrides
|
||||
// either way (GST-851); fails_when=a 4-core default runs lanes at once or the flag is ignored; why_new=no default existed; seam=laneConcurrency;
|
||||
// no_claim=the core count of any CI runner
|
||||
test('lane concurrency: serial on 4 cores or fewer, parallel above, --lanes overrides', () => {
|
||||
expect([laneConcurrency(undefined, 2), laneConcurrency(undefined, 4), laneConcurrency(undefined, 8)]).toEqual([1, 1, Infinity]);
|
||||
expect([laneConcurrency('serial', 64), laneConcurrency('parallel', 2)]).toEqual([1, Infinity]);
|
||||
expect(laneConcurrency('fast', 8)).toBe('--lanes fast is not serial or parallel');
|
||||
});
|
||||
|
||||
// Value: protects=Step 2's candidates and Step 1.4's carry check are reported for the agent to judge, and an open PR's carry verdict is printed; fails_when=a new bin/ artifact goes unmentioned or the carry check is skipped when a PR is open;
|
||||
// why_new=moved from Steps 1.4 and 2; seam=gh, gstack-ship-receipt carry; no_claim=whether a candidate is a distributable
|
||||
test('a new artifact path is a Step 2 candidate; an open PR runs the carry dry-run and prints its verdict', () => {
|
||||
|
||||
@@ -57,25 +57,26 @@ describe('write records the review depth at the checked-out head', () => {
|
||||
describe('the queue refuses a receipt below the depth its head needs', () => {
|
||||
// Value: protects=a receipt reviewed lighter than its head needs never READYs or seats, and the refusal says what to do;
|
||||
// fails_when=the queue trusts the recorded depth, or prints no required depth, reasons and inputs; seam=bindShippedReceipt
|
||||
test('light on a standard head, a missing depth from a current writer and an unbound gate scan refuse; a legacy receipt is accepted', () => {
|
||||
// R51: size never raises the depth, so the head that needs more than light is a risk-class one (a money path needs full).
|
||||
test('standard on a full (money) head, a missing depth from a current writer and an unbound gate scan refuse; a legacy receipt is accepted', () => {
|
||||
const repo = makeFixtureRepo(); repos.push(repo);
|
||||
const head = commit(repo, { 'lib/big.js': lines(80) });
|
||||
const head = commit(repo, { 'lib/billing.js': lines(80) });
|
||||
const body = path.join(repo.root, 'body.md');
|
||||
expect(writeReceipt(repo, body).status).toBe(EXIT.ok);
|
||||
expect(receiptIn(body).review_depth).toMatchObject({ depth: 'standard', runtime_lines: 80 });
|
||||
expect(receiptIn(body).review_depth).toMatchObject({ depth: 'full', runtime_lines: 80, risk: ['money'] });
|
||||
const base = repo.git('rev-parse', 'origin/main').slice(0, 12);
|
||||
const bind = (text: string) => bindShippedReceipt(repo.work, 'origin/main', text, '7', [head]);
|
||||
const good = bind(fs.readFileSync(body, 'utf8'));
|
||||
expect(good.ok).toBe(true);
|
||||
expect(good.ok && good.depthLine).toContain('REVIEW_DEPTH: receipt=standard required=standard runtime_lines=80 class=code');
|
||||
expect(good.ok && good.depthLine).toContain('REVIEW_DEPTH: receipt=full required=full runtime_lines=80 class=code');
|
||||
|
||||
const low = bind(rewrite(body, r => { r.review_depth.depth = 'light'; }));
|
||||
const low = bind(rewrite(body, r => { r.review_depth.depth = 'standard'; }));
|
||||
expect(low.ok).toBe(false);
|
||||
if (low.ok) return;
|
||||
expect(low.code).toBe('QUEUE_RECEIPT_DEPTH_RAISED');
|
||||
expect(low.depthLine).toContain('receipt=light required=standard runtime_lines=80');
|
||||
expect(low.depthLine).toContain('receipt=standard required=full runtime_lines=80');
|
||||
expect(low.depthLine).toContain(`inputs=base@${base},registry@${base},gate_scan@${head.slice(0, 12)}`);
|
||||
expect(low.message).toContain('runtime=80>50');
|
||||
expect(low.message).toContain('risk:money');
|
||||
expect(low.message).toContain(`the receipt is unchanged; run /ship on ${head.slice(0, 12)}`);
|
||||
|
||||
const unbound = bind(rewrite(body, r => { r.review_depth.depth = 'standard'; r.review_depth.gate_scan.head = 'a'.repeat(40); }));
|
||||
@@ -218,7 +219,8 @@ describe('the queue and a scoped second pass (GSTA-152 D6)', () => {
|
||||
expect(receiptIn(body).review_loop).toMatchObject({ passes: 2, dispatches: loop.dispatches });
|
||||
const bound = bindShippedReceipt(repo.work, 'origin/main', fs.readFileSync(body, 'utf8'), '7', [head]);
|
||||
expect(bound.ok).toBe(true);
|
||||
expect(bound.ok && bound.depthLine).toContain('REVIEW_DEPTH: receipt=standard required=standard');
|
||||
expect(bound.ok && bound.depthLine).toContain('REVIEW_DEPTH: receipt=light required=light');
|
||||
expect(bound.ok && bound.depthLine).toMatch(/ rounds=2$/);
|
||||
}, 60_000);
|
||||
});
|
||||
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"bytes": 66,
|
||||
"why": "GST-851: Step 5 prose says lanes run one at a time on 4 or fewer cores, names --lanes serial|parallel and the LANES line"
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
import { describe, test, expect, beforeEach, afterEach, afterAll } from 'bun:test';
|
||||
import { spawnSync } from 'child_process';
|
||||
import * as fs from 'fs';
|
||||
import * as os from 'os';
|
||||
import * as path from 'path';
|
||||
import { ceoOnlyEvidence, listFiles } from './helpers/autoplan-ceo-only-evidence';
|
||||
import { PTY_LONG_MS } from './helpers/eval-budgets';
|
||||
import { runSkillTest } from './helpers/session-runner';
|
||||
import { buildSeedConfig, seedHermeticGstackHome, seedHermeticRuntimeView } from './helpers/hermetic-env';
|
||||
import { FORCING_FLOOR_CEO } from './fixtures/forcing-finding-seeds';
|
||||
import {
|
||||
ROOT, runId, evalsEnabled, describeIfSelected, logCost, recordE2E, copyDirSync, createEvalCollector, finalizeEvalCollector,
|
||||
} from './helpers/e2e-helpers';
|
||||
|
||||
// /autoplan --ceo-only must finish a CEO review without asking anything (GST-537).
|
||||
// The seed is the plan that makes interactive /plan-ceo-review ask a question
|
||||
// (test/skill-e2e-plan-ceo-finding-floor.test.ts is the control on the same seed).
|
||||
// Every check is a pure function over the transcript and files,
|
||||
// proven against negative controls in test/autoplan-ceo-only.test.ts.
|
||||
|
||||
const evalCollector = createEvalCollector('e2e-autoplan-ceo-only');
|
||||
// Declared roots in the isolated HOME: the state root, the host's own config and temp, and
|
||||
// Bun's package cache (the runtime writes it under HOME on first use; run gsta215-e2e-2).
|
||||
const OWNED_HOME_ROOTS = ['gstack-home', '.claude', 'tmp', '.bun/install/cache', '.capy/git-shim.log'];
|
||||
|
||||
describeIfSelected('Autoplan CEO-only E2E', ['autoplan-ceo-only'], () => {
|
||||
let workDir = '';
|
||||
let stateDir = '';
|
||||
let env: Record<string, string> = {};
|
||||
let homeBefore: string[] = [];
|
||||
|
||||
const cleanup = () => {
|
||||
for (const dir of [workDir, stateDir]) if (dir) fs.rmSync(dir, { recursive: true, force: true });
|
||||
};
|
||||
|
||||
beforeEach(() => {
|
||||
try {
|
||||
workDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-autoplan-ceo-only-'));
|
||||
stateDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-autoplan-ceo-only-state-'));
|
||||
const configDir = path.join(stateDir, '.claude');
|
||||
const gstackHome = path.join(stateDir, 'gstack-home');
|
||||
const tempDir = path.join(stateDir, 'tmp');
|
||||
fs.mkdirSync(configDir);
|
||||
fs.mkdirSync(gstackHome);
|
||||
fs.mkdirSync(tempDir, { mode: 0o700 });
|
||||
fs.writeFileSync(path.join(configDir, '.claude.json'), JSON.stringify(buildSeedConfig({
|
||||
apiKey: process.env.ANTHROPIC_API_KEY ?? process.env.GSTACK_ANTHROPIC_API_KEY, trustedDirs: [workDir],
|
||||
})), { mode: 0o600 });
|
||||
seedHermeticGstackHome(gstackHome);
|
||||
fs.mkdirSync(path.join(configDir, 'skills'));
|
||||
seedHermeticRuntimeView(ROOT, path.join(configDir, 'skills', 'gstack'));
|
||||
env = { HOME: stateDir, CLAUDE_CONFIG_DIR: configDir, GSTACK_HOME: gstackHome, GSTACK_STATE_ROOT: gstackHome,
|
||||
TMPDIR: tempDir, TEMP: tempDir, TMP: tempDir };
|
||||
|
||||
const git = (...args: string[]) => {
|
||||
const r = spawnSync('git', args, { cwd: workDir, encoding: 'utf8', timeout: 10_000 });
|
||||
if (r.error || r.status !== 0) throw new Error(`CEO-only fixture setup failed: ${r.error?.message ?? r.stderr}`);
|
||||
};
|
||||
git('init', '-b', 'main');
|
||||
git('config', 'user.email', 'test@test.com');
|
||||
git('config', 'user.name', 'Test');
|
||||
// The seed's first line asks for a /tmp plan copy; the plan itself is the rest.
|
||||
fs.writeFileSync(path.join(workDir, 'PLAN.md'), FORCING_FLOOR_CEO.split('\n').slice(2).join('\n') + '\n');
|
||||
fs.writeFileSync(path.join(workDir, 'README.md'), '# Acme API\n\nA paid HTTP API with one pricing tier.\n');
|
||||
git('add', '.');
|
||||
git('-c', 'commit.gpgsign=false', 'commit', '--no-verify', '-m', 'Plan');
|
||||
for (const skill of ['autoplan', 'plan-ceo-review']) {
|
||||
const dest = path.join(workDir, '.claude', 'skills', skill);
|
||||
fs.mkdirSync(dest, { recursive: true });
|
||||
fs.copyFileSync(path.join(ROOT, skill, 'SKILL.md'), path.join(dest, 'SKILL.md'));
|
||||
copyDirSync(path.join(ROOT, skill, 'sections'), path.join(dest, 'sections'));
|
||||
}
|
||||
homeBefore = listFiles(stateDir, OWNED_HOME_ROOTS);
|
||||
} catch (error) {
|
||||
cleanup();
|
||||
throw error;
|
||||
}
|
||||
});
|
||||
|
||||
afterEach(cleanup);
|
||||
afterAll(() => finalizeEvalCollector(evalCollector));
|
||||
|
||||
// Titled by its E2E id so scripts/lib/paid-cases.ts needs no CASE_TEST_NAMES entry.
|
||||
test.skipIf(!evalsEnabled)('autoplan-ceo-only', async () => {
|
||||
const result = await runSkillTest({
|
||||
testName: 'autoplan-ceo-only',
|
||||
workingDirectory: workDir,
|
||||
env,
|
||||
prompt: '/autoplan --ceo-only --no-outside\n\nReview the plan in PLAN.md.',
|
||||
timeout: PTY_LONG_MS,
|
||||
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob', 'Agent', 'Skill'],
|
||||
maxTurns: 200,
|
||||
runId,
|
||||
});
|
||||
const before = new Set(homeBefore);
|
||||
const evidence = ceoOnlyEvidence({
|
||||
transcript: Array.isArray(result.transcript) ? result.transcript : [],
|
||||
output: result.output,
|
||||
strayWrites: listFiles(stateDir, OWNED_HOME_ROOTS).filter(f => !before.has(f)),
|
||||
mustMention: [/developer/i, /talk|interview|validat|evidence|research/i],
|
||||
});
|
||||
logCost('autoplan-ceo-only', result);
|
||||
recordE2E(evalCollector, 'autoplan-ceo-only', 'Autoplan CEO-only E2E', result, {
|
||||
passed: evidence.passed, ...(evidence.reasons.length ? { error: evidence.reasons.join('; ') } : {}),
|
||||
});
|
||||
expect(evidence.reasons).toEqual([]);
|
||||
}, PTY_LONG_MS + 60_000);
|
||||
});
|
||||
@@ -0,0 +1,65 @@
|
||||
/**
|
||||
* #3628: a host that keeps the Stop hook's stdin open after writing its payload
|
||||
* held the hook for as long as the pipe stayed open (fs.readFileSync(0) waits
|
||||
* for EOF), well past the hook's 2 s budget and toward the host's 15 s RPC
|
||||
* window. The hook must return within its budget and still repair the timeline.
|
||||
*
|
||||
* Windows-safe by construction: the hook runs through Bun argv
|
||||
* (process.execPath), state lives under os.tmpdir().
|
||||
*/
|
||||
import { afterEach, beforeEach, expect, test } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { bunIsolationArgs } from '../lib/bun-isolation';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const HOOK_TS = path.join(ROOT, 'hosts', 'claude', 'hooks', 'timeline-stop-hook.ts');
|
||||
const SLUG = 'stop-hook-stdin';
|
||||
const BUDGET_MS = 2_000;
|
||||
const STARTUP_ALLOWANCE_MS = process.platform === 'win32' ? 3_000 : 1_500;
|
||||
|
||||
let home = '', project = '', timeline = '';
|
||||
|
||||
beforeEach(() => {
|
||||
home = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-stop-stdin-home-')));
|
||||
project = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-stop-stdin-proj-')));
|
||||
fs.mkdirSync(path.join(home, 'projects', SLUG), { recursive: true });
|
||||
timeline = path.join(home, 'projects', SLUG, 'timeline.jsonl');
|
||||
fs.writeFileSync(timeline, JSON.stringify({ skill: 'review', event: 'started', session: '1-1' }) + '\n');
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
fs.rmSync(home, { recursive: true, force: true });
|
||||
fs.rmSync(project, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
async function runHookHoldingStdin(payload: string | null): Promise<{ code: number | null; ms: number }> {
|
||||
const started = performance.now();
|
||||
const child = Bun.spawn([process.execPath, ...bunIsolationArgs(), HOOK_TS], {
|
||||
stdin: 'pipe', stdout: 'ignore', stderr: 'ignore',
|
||||
env: { ...process.env, GSTACK_HOME: home, GSTACK_STATE_ROOT: home, GSTACK_PROJECT_SLUG: SLUG },
|
||||
});
|
||||
if (payload !== null) { child.stdin.write(payload); child.stdin.flush(); }
|
||||
const kill = setTimeout(() => child.kill(), 10_000);
|
||||
const code = await child.exited;
|
||||
clearTimeout(kill);
|
||||
const ms = performance.now() - started;
|
||||
try { child.stdin.end(); } catch { /* already closed */ }
|
||||
return { code, ms };
|
||||
}
|
||||
|
||||
const repairs = () => fs.readFileSync(timeline, 'utf8').split('\n').filter(Boolean).map(l => JSON.parse(l)).filter(e => e.source === 'stop-hook');
|
||||
|
||||
test('a complete payload on a pipe that never closes: returns within budget and repairs (#3628)', async () => {
|
||||
const r = await runHookHoldingStdin(JSON.stringify({ hook_event_name: 'Stop', cwd: project }));
|
||||
expect(r.code).toBe(0);
|
||||
expect(r.ms).toBeLessThan(BUDGET_MS + STARTUP_ALLOWANCE_MS);
|
||||
expect(repairs()).toHaveLength(1);
|
||||
}, 20_000);
|
||||
|
||||
test('an open pipe with no data: returns within budget (#3628)', async () => {
|
||||
const r = await runHookHoldingStdin(null);
|
||||
expect(r.code).toBe(0);
|
||||
expect(r.ms).toBeLessThan(BUDGET_MS + STARTUP_ALLOWANCE_MS);
|
||||
}, 20_000);
|
||||
@@ -310,8 +310,10 @@ describe('selectTests', () => {
|
||||
// The formerly keyless periodic AUQ probes drive the CEO skill too.
|
||||
expect(result.selected).toContain('auq-consistency');
|
||||
expect(result.selected).toContain('auq-verbose-vs-carved-ab');
|
||||
expect(result.selected.length).toBe(21);
|
||||
expect(result.skipped.length).toBe(Object.keys(E2E_TOUCHFILES).length - 21);
|
||||
// /autoplan --ceo-only runs plan-ceo-review's Phase 1 and its 0E mode rule.
|
||||
expect(result.selected).toContain('autoplan-ceo-only');
|
||||
expect(result.selected.length).toBe(22);
|
||||
expect(result.skipped.length).toBe(Object.keys(E2E_TOUCHFILES).length - 22);
|
||||
});
|
||||
|
||||
test('global touchfile triggers ALL tests', () => {
|
||||
|
||||
Reference in new issue
Block a user