mirror of
https://github.com/garrytan/gstack.git
synced 2026-10-31 22:14:25 +01:00
* fix(plan-eng-review): define the three references its judged text left dangling (GSTA-91) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix: pre-landing review fixes Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix: pre-landing review fixes Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * chore: ship coverage test, unattended doc and CHANGELOG for review fixes test/plan-reality.test.ts asserts each phase's gstack-plan-reality check line names PLAN_REALITY (Step 7 gap). docs/unattended.md names the unattended-session block the skill prose now cites (Step 14.5 audit). CHANGELOG [Unreleased] covers the hook pointer fix and the new tests. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * docs(changelog): drop the hook directive claim (GSTA-91) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix: keep destructive options pending in the unattended AUQ fallback (GSTA-91) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix: pre-landing review fixes Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix: trim the unattended pointers so every size pin goes down (GSTA-91) The failure fallback's unattended branch now says a consent, destructive or irreversible option, unrecommended question or approval gate stays pending, keeping the carve-out. Item 1 drops its dangling '(per its printed unattended-session block)' pointer. CQ3 now asserts no section or block pointer remains in item 1; CQ4 is retitled to the block and hook; the plan-reality pointer loop drops devex, which renders no reality rows. Every host and the /ship goldens are regenerated and all 89 pins go down versus4814b3d8a(78 by 62, 11 by 38). Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix: pre-landing review fixes Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * docs(unattended): drop the claim that skill prose cites the block id (GSTA-91) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix: pre-landing review fixes Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix: pre-landing review fixes Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix: item 1 names destructive or irreversible options as pending in unattended runs (GSTA-91) Dropping item 1's block pointer also dropped its only route to the printed block's destructive/irreversible carve-out. Item 1 now lists them in its unattended pending set, matching the failure-fallback branch, the hook and the printed block; the shared Exception is unchanged. CQ3 requires destructive and irreversible. Regenerated every host and the /ship goldens; every pin still goes down versus4814b3d8a(78 by 32, 11 by 8). Includes the /ship Step 7 test that docs/unattended.md names the printed block header and END. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --------- Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
397 lines
27 KiB
TypeScript
397 lines
27 KiB
TypeScript
/**
|
||
* Plan D1: reality rows in plan reviews. Pins the row ids, the `REALITY:` line
|
||
* grammar, the checker's missing/unreceipted/n/a branches, the bin's exit codes
|
||
* and `PLAN_REALITY:` line, the unattended `incomplete` verdict, and that the
|
||
* rendered plan-review skills and autoplan phase files carry the contract.
|
||
* Each tier 1 row has a positive fixture from this wave's own review record
|
||
* (test/fixtures/multi-agent-wave/reality-rows.json, receipts verified against
|
||
* the record) and a decoy that must not count. Prose is not pinned.
|
||
*/
|
||
import { describe, expect, test } from 'bun:test';
|
||
import { spawnSync } from 'node:child_process';
|
||
import * as fs from 'node:fs';
|
||
import * as os from 'node:os';
|
||
import * as path from 'node:path';
|
||
import { EXIT } from '../lib/headless-artifacts';
|
||
import { PLAN_REALITY_CHECKER, REALITY_ROWS, REALITY_SCOPES, TIER1_ROW_IDS, checkPlanReality, checkRealityRows, parseRealityLines, parseScopes, renderIncidentTable, renderRows, requiredRows, vaguePlanTerms } from '../lib/plan-reality';
|
||
import { extractHeading } from '../lib/autoplan-prompts';
|
||
import { RESULT_CODES } from '../lib/result-codes';
|
||
import { generatePlanRealityRows } from '../scripts/resolvers/plan-reality';
|
||
import type { TemplateContext } from '../scripts/resolvers/types';
|
||
|
||
const ROOT = path.resolve(import.meta.dir, '..');
|
||
const BIN = path.join(ROOT, 'bin', 'gstack-plan-reality');
|
||
const FIXTURE = JSON.parse(fs.readFileSync(path.join(ROOT, 'test', 'fixtures', 'multi-agent-wave', 'reality-rows.json'), 'utf8')) as {
|
||
rows: Record<string, { positive: string; source: { file: string; lines: [number, number]; contains: string }; decoy: string; decoy_kind: string; metric?: Record<string, string> }>;
|
||
};
|
||
const positives = Object.values(FIXTURE.rows).map(r => r.positive).join('\n');
|
||
|
||
function run(args: string[], input?: string, env: Record<string, string> = {}) {
|
||
const r = spawnSync('bun', [BIN, ...args], { cwd: ROOT, encoding: 'utf8', timeout: 30_000, input, env: { ...process.env, GSTACK_SESSION_KIND: '', ...env } });
|
||
return { code: r.status, out: r.stdout, err: r.stderr };
|
||
}
|
||
|
||
describe('reality row table', () => {
|
||
test('seven tier 1 rows, every row has a check and an incident, ids are unique and scopes known', () => {
|
||
expect(TIER1_ROW_IDS).toEqual(['premise-table', 'already-done', 'surface-check', 'binding-decisions', 'numbers', 'deferred-asks', 'pr-count']);
|
||
expect(REALITY_ROWS.filter(r => r.tier === 2).length).toBe(11);
|
||
expect(new Set(REALITY_ROWS.map(r => r.id)).size).toBe(REALITY_ROWS.length);
|
||
for (const r of REALITY_ROWS) {
|
||
expect(r.check.length).toBeGreaterThan(20);
|
||
expect(r.incident.length).toBeGreaterThan(20);
|
||
expect(REALITY_SCOPES).toContain(r.scope);
|
||
expect(r.tier === 1 ? 'always' : 'scoped').toBe(r.scope === 'always' ? 'always' : 'scoped');
|
||
}
|
||
});
|
||
test('required rows: tier 1 always, tier 2 only for a detected scope', () => {
|
||
expect(requiredRows([]).map(r => r.id)).toEqual([...TIER1_ROW_IDS]);
|
||
expect(requiredRows(['db']).map(r => r.id)).toContain('transaction-semantics');
|
||
expect(requiredRows(['db']).map(r => r.id)).not.toContain('recall');
|
||
expect(parseScopes('db, search,bogus')).toEqual({ scopes: ['db', 'search'], unknown: ['bogus'] });
|
||
});
|
||
});
|
||
|
||
describe('REALITY: line grammar', () => {
|
||
test('parses id, status, receipt (path:line, path:line-line, sha) and key=value metrics; tolerates list and table prefixes', () => {
|
||
const lines = parseRealityLines([
|
||
'REALITY: premise-table finding 27 checked unverified=3 docs/x/ceo-native.md:11-37',
|
||
'- REALITY: already-done pass 0 hits 91bbd9e',
|
||
'| REALITY: surface-check finding bun upgrade bin/gstack-capy-install:52 |',
|
||
'REALITY: numbers pass 7 PRs 48 min',
|
||
'REALITY: transaction-semantics n/a scope db absent',
|
||
'reality: not-a-row pass lower-case prefix is not the grammar',
|
||
].join('\n'));
|
||
expect(lines.map(l => [l.id, l.status, l.receipt])).toEqual([
|
||
['premise-table', 'finding', 'docs/x/ceo-native.md:11-37'],
|
||
['already-done', 'pass', '91bbd9e'],
|
||
['surface-check', 'finding', 'bin/gstack-capy-install:52'],
|
||
['numbers', 'pass', null],
|
||
['transaction-semantics', 'n/a', null],
|
||
]);
|
||
expect(lines[0]!.metrics.unverified).toBe('3');
|
||
});
|
||
});
|
||
|
||
describe('checkRealityRows', () => {
|
||
test('every tier 1 positive fixture from the review record passes and the premise metric is scored', () => {
|
||
const r = checkRealityRows(positives, { phase: 'ceo', sessionKind: 'unattended' });
|
||
expect(r.missing).toEqual([]);
|
||
expect(r.unreceipted).toEqual([]);
|
||
expect(r.verdict).toBe('complete');
|
||
expect(r.metrics).toEqual({ unverified: '3' });
|
||
expect(r.line).toBe('PLAN_REALITY: phase=ceo rows=7/7 missing=none unverified=3 verdict=complete');
|
||
});
|
||
test('each positive receipt points into the review record at the stated lines', () => {
|
||
for (const [id, row] of Object.entries(FIXTURE.rows)) {
|
||
const text = fs.readFileSync(path.join(ROOT, row.source.file), 'utf8').split('\n');
|
||
const [from, to] = row.source.lines;
|
||
const excerpt = text.slice(from - 1, to).join('\n');
|
||
expect(excerpt, `${id}: ${row.source.file}:${from}-${to}`).toContain(row.source.contains);
|
||
const parsed = parseRealityLines(row.positive)[0]!;
|
||
expect(parsed.id).toBe(id);
|
||
expect(parsed.receipt, `${id} carries a receipt`).not.toBeNull();
|
||
expect(row.positive).toContain(`${row.source.file}:${from}`);
|
||
}
|
||
});
|
||
test('each decoy fails for its own row while the other six positives stand (negative control per row)', () => {
|
||
for (const [id, row] of Object.entries(FIXTURE.rows)) {
|
||
const others = Object.entries(FIXTURE.rows).filter(([k]) => k !== id).map(([, r]) => r.positive);
|
||
const r = checkRealityRows([...others, row.decoy].join('\n'), { phase: 'eng' });
|
||
expect([...r.missing, ...r.unreceipted], `${id} decoy (${row.decoy_kind}) must not count`).toEqual([id]);
|
||
expect(r.verdict).toBe('repair');
|
||
}
|
||
});
|
||
test('a review that only talks about the rows in prose has every row missing', () => {
|
||
const prose = 'The premises were checked and already-done work was considered. Surface check done. Numbers estimated.';
|
||
const r = checkRealityRows(prose, { phase: 'ceo', sessionKind: 'unattended' });
|
||
expect(r.missing).toEqual([...TIER1_ROW_IDS]);
|
||
expect(r.verdict).toBe('incomplete');
|
||
expect(r.line).toContain('rows=0/7');
|
||
});
|
||
test('verdict word: unattended → incomplete, anything else → repair; complete needs no session kind', () => {
|
||
const partial = positives.split('\n').slice(0, 6).join('\n');
|
||
expect(checkRealityRows(partial, { phase: 'eng', sessionKind: 'unattended' }).verdict).toBe('incomplete');
|
||
expect(checkRealityRows(partial, { phase: 'eng', sessionKind: 'interactive' }).verdict).toBe('repair');
|
||
expect(checkRealityRows(partial, { phase: 'eng' }).verdict).toBe('repair');
|
||
expect(checkRealityRows(partial, { phase: 'eng' }).missing).toEqual(['pr-count']);
|
||
});
|
||
test('tier 2 rows: required only under their scope; n/a is accepted off-scope and a tier 1 n/a is missing', () => {
|
||
const withNa = `${positives}\nREALITY: transaction-semantics n/a no db change`;
|
||
expect(checkRealityRows(withNa, { phase: 'eng' }).missing).toEqual([]);
|
||
expect(checkRealityRows(withNa, { phase: 'eng', scopes: ['db'] }).missing).toEqual(['transaction-semantics']);
|
||
const dbRow = `${positives}\nREALITY: transaction-semantics pass idle, pooled and reserved traced lib/pool.ts:40-88`;
|
||
expect(checkRealityRows(dbRow, { phase: 'eng', scopes: ['db'] }).line).toContain('rows=8/8');
|
||
expect(checkRealityRows(positives.replace(/^REALITY: numbers finding/m, 'REALITY: numbers n/a'), { phase: 'eng' }).missing).toEqual(['numbers']);
|
||
});
|
||
test('a later real line outranks an earlier n/a for the same row; unknown row ids are reported, not counted', () => {
|
||
const r = checkRealityRows(`REALITY: already-done n/a\n${positives}\nREALITY: premises pass lookalike plan.md:1`, { phase: 'ceo' });
|
||
expect(r.missing).toEqual([]);
|
||
expect(r.unknownRows).toEqual(['premises']);
|
||
});
|
||
});
|
||
|
||
describe('gstack-plan-reality bin', () => {
|
||
test('--help prints the exit table and the line shape; no command is usage', () => {
|
||
const help = run(['--help']);
|
||
expect(help.code).toBe(EXIT.ok);
|
||
expect(help.out).toContain('Exit codes: 0 ok · 1 fail · 2 usage · 3 refused or needs a flag');
|
||
expect(help.out).toContain('REALITY: <row> pass|finding|n/a <summary> <file:line>');
|
||
expect(run([]).code).toBe(EXIT.usage);
|
||
expect(run(['rows']).code).toBe(EXIT.usage);
|
||
expect(run(['rows', '--phase', 'eng', '--scope', 'bogus']).code).toBe(EXIT.usage);
|
||
expect(run(['frob', '--phase', 'eng']).code).toBe(EXIT.usage);
|
||
});
|
||
test('rows prints the applicable rows; --scope adds tier 2; --json is JSON only', () => {
|
||
const plain = run(['rows', '--phase', 'eng']);
|
||
expect(plain.code).toBe(EXIT.ok);
|
||
for (const id of TIER1_ROW_IDS) expect(plain.out).toContain(`- ${id} (tier 1)`);
|
||
expect(plain.out).not.toContain('recall');
|
||
expect(run(['rows', '--phase', 'eng', '--scope', 'search']).out).toContain('- recall (tier 2, scope search)');
|
||
expect(run(['rows', '--phase', 'eng', '--deep']).out).toContain('Incident:');
|
||
const json = JSON.parse(run(['rows', '--phase', 'ceo', '--json']).out);
|
||
expect(json.rows.map((r: { id: string }) => r.id)).toEqual([...TIER1_ROW_IDS]);
|
||
});
|
||
test('check: complete exits 0; a miss exits 1 with REALITY_ROW_MISSING; unattended prints verdict=incomplete; stdin and --json', () => {
|
||
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-plan-reality-'));
|
||
const file = path.join(tmp, 'review.md');
|
||
fs.writeFileSync(file, `prose\n${positives}\n`);
|
||
const ok = run(['check', '--phase', 'ceo', file]);
|
||
expect(ok.code).toBe(EXIT.ok);
|
||
expect(ok.out.trim()).toBe('PLAN_REALITY: phase=ceo rows=7/7 missing=none unverified=3 verdict=complete');
|
||
|
||
const partial = positives.split('\n').slice(1).join('\n');
|
||
const miss = run(['check', '--phase', 'eng', '-'], partial, { GSTACK_SESSION_KIND: 'unattended' });
|
||
expect(miss.code).toBe(EXIT.fail);
|
||
expect(miss.out).toContain('missing=premise-table verdict=incomplete');
|
||
expect(miss.err).toContain('(REALITY_ROW_MISSING)');
|
||
expect(miss.err).toContain('fix:');
|
||
|
||
const flag = run(['check', '--phase', 'eng', '-', '--session-kind', 'unattended'], partial);
|
||
expect(flag.out).toContain('verdict=incomplete');
|
||
const interactive = run(['check', '--phase', 'eng', '-'], partial);
|
||
expect(interactive.out).toContain('verdict=repair');
|
||
|
||
const unreceipted = run(['check', '--phase', 'ceo', '-', '--json'], `${partial}\n${FIXTURE.rows['premise-table']!.decoy}`);
|
||
expect(unreceipted.code).toBe(EXIT.fail);
|
||
const parsed = JSON.parse(unreceipted.out);
|
||
expect(parsed.unreceipted).toEqual(['premise-table']);
|
||
expect(unreceipted.err).toContain('(REALITY_RECEIPT_MISSING)');
|
||
const unreceiptedText = run(['check', '--phase', 'ceo', '-'], `${partial}\n${FIXTURE.rows['premise-table']!.decoy}`);
|
||
expect(unreceiptedText.out).toMatch(/^PLAN_REALITY: phase=ceo .*\bunreceipted=premise-table\b.* verdict=repair$/m);
|
||
expect(run(['check', '--phase', 'ceo', path.join(tmp, 'absent.md')]).code).toBe(EXIT.fail);
|
||
fs.rmSync(tmp, { recursive: true, force: true });
|
||
});
|
||
});
|
||
|
||
describe('captured tier 1 n/a (plan-mode-no-op, paid run 39004466422)', () => {
|
||
// The named-target plan-eng-review session marked deferred-asks n/a, check returned repair, and the repair loop ran
|
||
// past the 300 s capture before the first question.
|
||
const captured = 'REALITY: deferred-asks n/a no linked issue; the plan cites no request to compare against package.json:2';
|
||
test('check rejects the captured line as a missing row, and the skill text forbids it', () => {
|
||
const others = Object.entries(FIXTURE.rows).filter(([id]) => id !== 'deferred-asks').map(([, row]) => row.positive).join('\n');
|
||
const rejected = run(['check', '--phase', 'eng', '-'], `${others}\n${captured}\n`);
|
||
expect(rejected.code).toBe(EXIT.fail);
|
||
expect(rejected.out).toMatch(/^PLAN_REALITY: phase=eng rows=6\/7 missing=deferred-asks .*verdict=repair$/m);
|
||
expect(rejected.err).toContain('(REALITY_ROW_MISSING)');
|
||
const repaired = run(['check', '--phase', 'eng', '-'], `${others}\n${captured.replace(' n/a ', ' pass ')}\n`);
|
||
expect(repaired.out).toMatch(/^PLAN_REALITY: phase=eng rows=7\/7 missing=none .*verdict=complete$/m);
|
||
const rule = 'tier 1 always, never `n/a`: premise-table, already-done, surface-check, binding-decisions, numbers, deferred-asks, pr-count; tier 2 by detected scope, `--scope <csv>`). Every row cites a file:line receipt (with no linked issue, deferred-asks is `pass`)';
|
||
for (const phase of ['plan-eng-review', 'plan-ceo-review', 'plan-devex-review']) {
|
||
expect(generatePlanRealityRows({ skillName: phase, host: 'claude', paths: {} } as unknown as TemplateContext).replace(/\s+/g, ' ')).toContain(rule);
|
||
}
|
||
for (const rel of ['plan-eng-review/sections/review-sections.md', 'plan-ceo-review/SKILL.md']) {
|
||
expect(fs.readFileSync(path.join(ROOT, rel), 'utf8').replace(/\s+/g, ' ')).toContain(rule);
|
||
}
|
||
});
|
||
});
|
||
|
||
describe('rendered templates carry the contract', () => {
|
||
const read = (rel: string) => fs.readFileSync(path.join(ROOT, rel), 'utf8');
|
||
const ctx = (skillName: string) => ({ skillName, host: 'claude', paths: {} } as unknown as TemplateContext);
|
||
test('resolver variants name the bin, the tier 1 rows, the line shape and the unattended rule', () => {
|
||
const eng = generatePlanRealityRows(ctx('plan-eng-review'));
|
||
expect(eng).toContain('gstack-plan-reality rows --phase eng');
|
||
expect(eng).toContain('gstack-plan-reality check --phase eng');
|
||
for (const id of TIER1_ROW_IDS) expect(eng).toContain(id);
|
||
expect(eng).toContain('REALITY: <row> pass|finding|n/a <summary> <file:line>');
|
||
expect(generatePlanRealityRows(ctx('plan-ceo-review'))).toContain('--phase ceo');
|
||
expect(generatePlanRealityRows(ctx('plan-devex-review'))).toContain('--phase dx');
|
||
// Generated by /ship Step 7 coverage audit (GSTA-91, PR #3421).
|
||
// Value: protects=the operative block says its check prints the PLAN_REALITY line the summary row asks for;
|
||
// fails_when=that pointer is dropped and PLAN_REALITY is undefined in the block; why_new=only the summary variant
|
||
// was asserted to name PLAN_REALITY; seam=none; no_claim=the bin's output (pinned by the check tests above)
|
||
for (const skill of ['plan-eng-review', 'plan-ceo-review']) {
|
||
expect(generatePlanRealityRows(ctx(skill)), skill).toMatch(/gstack-plan-reality check --phase [a-z]+ [^\n]*`PLAN_REALITY`/);
|
||
}
|
||
const deep = generatePlanRealityRows(ctx('plan-eng-review'), ['deep']);
|
||
for (const r of REALITY_ROWS) expect(deep).toContain(`| \`${r.id}\` | ${r.tier} | ${r.scope} |`);
|
||
expect(deep).toBe(deep.replace(/\n{3,}/g, '\n\n'));
|
||
expect(generatePlanRealityRows(ctx('autoplan'), ['autoplan', 'eng'])).toContain('check --phase eng');
|
||
expect(generatePlanRealityRows(ctx('autoplan'), ['pregate'])).toContain('`incomplete`');
|
||
expect(generatePlanRealityRows(ctx('plan-eng-review'), ['summary'])).toContain('PLAN_REALITY');
|
||
expect(renderIncidentTable(REALITY_ROWS).split('\n').length).toBe(REALITY_ROWS.length + 2);
|
||
expect(renderRows(requiredRows([]), { deep: false }).split('\n').length).toBe(TIER1_ROW_IDS.length);
|
||
});
|
||
test('plan-eng-review: Scope Challenge block, deep incident table and Completion summary line', () => {
|
||
const sections = read('plan-eng-review/sections/review-sections.md');
|
||
expect(sections).toContain('gstack-plan-reality rows --phase eng');
|
||
expect(sections).toContain('### Reality rows (why each exists)');
|
||
expect(sections).toContain('| `pr-count` | 1 | always |');
|
||
expect(sections).toContain('- Reality rows: ___/___ applicable rows emitted (`PLAN_REALITY` line), ___ unverified premises');
|
||
expect(read('plan-eng-review/sections/checklist.md')).toContain('| Reality rows (`REALITY:` lines, `PLAN_REALITY` check) | Reality rows (why each exists) | always |');
|
||
});
|
||
test('plan-ceo-review: 0A/0B block in the skeleton, Completion Summary row and checklist row', () => {
|
||
const skeleton = read('plan-ceo-review/SKILL.md');
|
||
const at = skeleton.indexOf('gstack-plan-reality rows --phase ceo');
|
||
expect(at).toBeGreaterThan(skeleton.indexOf('### 0B. Existing Code Leverage'));
|
||
expect(at).toBeLessThan(skeleton.indexOf('### 0C. Dream State Mapping'));
|
||
expect(read('plan-ceo-review/sections/review-sections.md')).toContain('| Reality rows | ___/___ rows (PLAN_REALITY), ___ unverified |');
|
||
expect(read('plan-ceo-review/sections/review-sections.md')).toContain('### Reality rows\n');
|
||
expect(read('plan-ceo-review/sections/checklist.md')).toContain('| Reality rows (`REALITY:` lines, `PLAN_REALITY` check) | Reality rows | always |');
|
||
});
|
||
test('autoplan: CEO and Eng phase files run the check; the Pre-Gate names incomplete for unattended misses', () => {
|
||
expect(read('autoplan/sections/ceo-phase.md')).toContain('gstack-plan-reality check --phase ceo');
|
||
expect(read('autoplan/sections/eng-phase.md')).toContain('gstack-plan-reality check --phase eng');
|
||
const skeleton = read('autoplan/SKILL.md');
|
||
const pregate = skeleton.slice(skeleton.indexOf('## Pre-Gate Verification'), skeleton.indexOf('## Phase 4: Final Approval Gate'));
|
||
expect(pregate).toContain('`REALITY:` lines');
|
||
expect(pregate).toContain('verdict=complete');
|
||
expect(pregate).toContain('`incomplete`');
|
||
});
|
||
test('the two result codes have troubleshooting anchors', () => {
|
||
const doc = read('docs/troubleshooting.md');
|
||
for (const code of ['REALITY_ROW_MISSING', 'REALITY_RECEIPT_MISSING'] as const) {
|
||
expect(doc).toContain(`<a id="${RESULT_CODES[code].anchor}"></a>`);
|
||
}
|
||
});
|
||
});
|
||
|
||
describe('field wave item 12: a receipt per measured number, the rule text for a rule claim', () => {
|
||
const others = (id: string) => Object.entries(FIXTURE.rows).filter(([k]) => k !== id).map(([, r]) => r.positive).join('\n');
|
||
const FIVE = 'REALITY: numbers pass p50 120ms, p95 340ms, 13% faster, 28k tokens, cost $41 [measured: test/bench.ts:10]';
|
||
|
||
test('five measured numbers sharing one receipt leave four unreceipted, each printed with its clause', () => {
|
||
const r = checkRealityRows(`${others('numbers')}\n${FIVE}`, { phase: 'eng', sessionKind: 'unattended' });
|
||
expect(r.missing).toEqual([]);
|
||
expect(r.unreceipted).toEqual([]);
|
||
expect(r.unreceipted_numbers).toEqual(['p50 120ms', 'p95 340ms', '13% faster', '28k tokens']);
|
||
expect(r.numberLines).toEqual(['NUMBER_UNRECEIPTED: p50 120ms', 'NUMBER_UNRECEIPTED: p95 340ms', 'NUMBER_UNRECEIPTED: 13% faster', 'NUMBER_UNRECEIPTED: 28k tokens']);
|
||
expect(r.verdict).toBe('incomplete');
|
||
expect(r.line).toContain(' unreceipted_numbers=4 ');
|
||
expect(checkRealityRows(`${others('numbers')}\n${FIVE}`, { phase: 'eng' }).verdict).toBe('repair');
|
||
});
|
||
|
||
test('each clause with its own receipt (file:line, script path or a command in backticks) passes; unmeasured counts, versions and ids need none', () => {
|
||
const own = 'REALITY: numbers pass p95 340ms lib/x.ts:4, 13% faster `bun run bench --json`, 28k tokens scripts/count-tokens.ts, recall 0.82 vs 0.79 on gstack 1.91.95.0 per F2 and #3160 [estimated]';
|
||
const r = checkRealityRows(`${others('numbers')}\n${own}`, { phase: 'eng' });
|
||
expect(r.unreceipted_numbers).toEqual([]);
|
||
expect(r.verdict).toBe('complete');
|
||
});
|
||
|
||
test('a rule claim without the rule text is unquoted; the quoted rule or a file:line link to it passes', () => {
|
||
const bare = 'REALITY: binding-decisions pass the decision rule was written first 91bbd9e';
|
||
const r = checkRealityRows(`${others('binding-decisions')}\n${bare}`, { phase: 'eng', sessionKind: 'unattended' });
|
||
expect(r.unreceipted).toEqual([]);
|
||
expect(r.unquoted).toEqual([bare]);
|
||
expect(r.line).toContain(' unquoted=1 ');
|
||
expect(r.verdict).toBe('incomplete');
|
||
const quoted = 'REALITY: binding-decisions pass the decision rule was written first: "stop if recall drops below the base on any bucket" 91bbd9e';
|
||
expect(checkRealityRows(`${others('binding-decisions')}\n${quoted}`, { phase: 'eng' }).verdict).toBe('complete');
|
||
const linked = 'REALITY: binding-decisions pass the decision rule was written first, see docs/plan.md:4';
|
||
expect(checkRealityRows(`${others('binding-decisions')}\n${linked}`, { phase: 'eng' }).unquoted).toEqual([]);
|
||
});
|
||
|
||
test('receipts on dot paths count; a sha receipt needs a hex letter or a commit/backtick context', () => {
|
||
expect(parseRealityLines('REALITY: pr-count pass one PR .gstack/ship-policy.json:1')[0]!.receipt).toBe('.gstack/ship-policy.json:1');
|
||
expect(parseRealityLines('REALITY: pr-count pass ci lane .github/workflows/ci.yml:12-20')[0]!.receipt).toBe('.github/workflows/ci.yml:12-20');
|
||
expect(parseRealityLines('REALITY: numbers pass run took 20261012 cycles')[0]!.receipt).toBeNull();
|
||
expect(parseRealityLines('REALITY: already-done pass base clean at commit 1234567')[0]!.receipt).toBe('commit 1234567');
|
||
expect(parseRealityLines('REALITY: already-done pass base clean at `7654321` today')[0]!.receipt).toBe('`7654321`');
|
||
});
|
||
|
||
test('bin: NUMBER_UNRECEIPTED lines follow the PLAN_REALITY line and the verdict exits 1 with a fix', () => {
|
||
const r = run(['check', '--phase', 'eng', '-', '--session-kind', 'unattended'], `${others('numbers')}\n${FIVE}`);
|
||
expect(r.code).toBe(EXIT.fail);
|
||
expect(r.out.split('\n').filter(Boolean)).toEqual([
|
||
expect.stringMatching(/^PLAN_REALITY: phase=eng rows=7\/7 missing=none unreceipted_numbers=4 .*verdict=incomplete$/),
|
||
'NUMBER_UNRECEIPTED: p50 120ms', 'NUMBER_UNRECEIPTED: p95 340ms', 'NUMBER_UNRECEIPTED: 13% faster', 'NUMBER_UNRECEIPTED: 28k tokens',
|
||
]);
|
||
expect(r.err).toContain('(REALITY_RECEIPT_MISSING)');
|
||
expect(r.err).toContain('fix:');
|
||
});
|
||
|
||
test('checkPlanReality(planText): plan prose gets the same number and rule checks, pure and session-kind mapped', () => {
|
||
const plan = 'Cold install measured 32.5 s; the warm rerun takes 13 s (bin/gstack-capy-install:91).\n```\nlatency 900 ms in a fenced example\n```\nThe decision rule was written first.\nThe preregistered rule is below.\n> stop when recall drops below the base\n';
|
||
const r = checkPlanReality(plan, { sessionKind: 'unattended' });
|
||
expect(r.unreceipted_numbers).toEqual(['Cold install measured 32.5 s']);
|
||
expect(r.unquoted).toEqual(['The decision rule was written first.']);
|
||
expect(r.verdict).toBe('incomplete');
|
||
expect(r.line).toBe('PLAN_REALITY: scope=plan unreceipted_numbers=1 unquoted=1 verdict=incomplete');
|
||
expect(r.numberLines).toEqual(['NUMBER_UNRECEIPTED: Cold install measured 32.5 s']);
|
||
expect(checkPlanReality('Add a retry with a 30 s cap (lib/worker.ts:12).\n').line).toBe('PLAN_REALITY: scope=plan unreceipted_numbers=0 unquoted=0 verdict=complete');
|
||
expect(checkPlanReality(plan).verdict).toBe('repair');
|
||
});
|
||
|
||
test('false-positive bound: the field-wave plan itself raises only a small, pinned count', () => {
|
||
const text = fs.readFileSync(path.join(ROOT, 'docs', 'designs', 'GSTACK_FIELD_WAVE_2026_10_12.md'), 'utf8');
|
||
const r = checkPlanReality(extractHeading(text, 'Implementation plan')!);
|
||
expect(r.unquoted).toEqual([]);
|
||
expect(r.unreceipted_numbers.length).toBeGreaterThan(0);
|
||
expect(r.unreceipted_numbers.length).toBeLessThanOrEqual(6);
|
||
});
|
||
});
|
||
|
||
describe('next wave B6/B10: PLAN_REALITY flags without false positives; plan quotes need no receipt', () => {
|
||
const others = (id: string) => Object.entries(FIXTURE.rows).filter(([k]) => k !== id).map(([, r]) => r.positive).join('\n');
|
||
|
||
test('B6: item and section references are not quantities; a phrase in backticks is named, not claimed (plan scope)', () => {
|
||
const plan = "B1. `rec`: one word takes every recommendation (item 5).\nB6. When the plan names `minimum effect`, `margin`, `decision rule` or a dollar cost, the gate flags it (items 10, 14–19; section 3).\n";
|
||
const r = checkPlanReality(plan);
|
||
expect(r.unreceipted_numbers).toEqual([]);
|
||
expect(r.unquoted).toEqual([]);
|
||
expect(checkPlanReality('Startup takes roughly 9 s on a cold machine.\n').unreceipted_numbers).toEqual(['Startup takes roughly 9 s on a cold machine.']);
|
||
expect(checkPlanReality('The decision rule was written first.\n').unquoted).toEqual(['The decision rule was written first.']);
|
||
expect(PLAN_REALITY_CHECKER).toBe('plan-reality/3');
|
||
});
|
||
|
||
test('B-f negative control: a measured number inside backticks in review scope stays flagged; a command in backticks still receipts', () => {
|
||
const quiet = 'REALITY: numbers pass the cold install took `86 s` on Capy';
|
||
expect(checkRealityRows(`${others('numbers')}\n${quiet}`, { phase: 'eng' }).unreceipted_numbers).toEqual(['the cold install took `86 s` on Capy']);
|
||
const cmd = 'REALITY: numbers pass the cold install took 86 s `bun run bench --json` on Capy';
|
||
expect(checkRealityRows(`${others('numbers')}\n${cmd}`, { phase: 'eng' }).unreceipted_numbers).toEqual([]);
|
||
});
|
||
|
||
test('B10: in reviewer scope a number quoted verbatim from the reviewed plan needs no receipt; a reworded or unknown quote does', () => {
|
||
const plan = 'Add exponential backoff: base 500 ms, factor 2, cap 30 s, jitter 10%.\n';
|
||
const quoted = 'REALITY: numbers pass the plan says "base 500 ms, factor 2, cap 30 s" and gives no reason for the cap (plan.md:1)';
|
||
expect(checkRealityRows(`${others('numbers')}\n${quoted}`, { phase: 'eng', planText: plan }).unreceipted_numbers).toEqual([]);
|
||
expect(checkRealityRows(`${others('numbers')}\n${quoted}`, { phase: 'eng' }).unreceipted_numbers.length).toBeGreaterThan(0);
|
||
const reworded = 'REALITY: numbers pass the plan says "a 45 s cap" and gives no reason';
|
||
expect(checkRealityRows(`${others('numbers')}\n${reworded}`, { phase: 'eng', planText: plan }).unreceipted_numbers.length).toBe(1);
|
||
const own = 'REALITY: numbers pass the plan says "cap 30 s"; retries took 900 ms in my run';
|
||
expect(checkRealityRows(`${others('numbers')}\n${own}`, { phase: 'eng', planText: plan }).unreceipted_numbers).toEqual(['retries took 900 ms in my run']);
|
||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'plan-reality-quote-'));
|
||
fs.writeFileSync(path.join(dir, 'plan.md'), plan);
|
||
const bin = run(['check', '--phase', 'eng', '--plan', path.join(dir, 'plan.md'), '-'], `${others('numbers')}\n${quoted}`);
|
||
expect(bin.code, bin.err).toBe(EXIT.ok);
|
||
expect(run(['check', '--phase', 'eng', '-'], `${others('numbers')}\n${quoted}`).code).toBe(EXIT.fail);
|
||
fs.rmSync(dir, { recursive: true, force: true });
|
||
});
|
||
|
||
test('B6: vaguePlanTerms flags minimum effect, margin, decision rule and dollar cost only without a number, arm list or rule body', () => {
|
||
const plan = [
|
||
'Ship when the minimum effect is reached.',
|
||
'Keep a safety margin on the spend.',
|
||
'A margin of 15% over the pilot price.',
|
||
'The decision rule:',
|
||
'- arm A: keep the default',
|
||
'- arm B: switch',
|
||
'We price the dollar cost later (item 9).',
|
||
'The decision rule is "stop when recall drops below the base".',
|
||
'Name `minimum effect` in the template.',
|
||
'```', 'margin inside a fence', '```',
|
||
].join('\n');
|
||
expect(vaguePlanTerms(plan).map(v => [v.phrase, v.line])).toEqual([['minimum effect', 1], ['margin', 2], ['dollar cost', 7]]);
|
||
});
|
||
});
|