mirror of
https://github.com/garrytan/gstack.git
synced 2026-10-03 01:46:55 +02:00
test(eng-batching): bind unsourced native briefs through the report's target
Run 36606688266 asked ten separate native review questions (D1-D9 bound to ledger records R1-R9) and failed reviewCount=0 < FLOOR=3: its briefs named the plan by title instead of citing PLAN.md, its report declared 'Review target (fixed): PLAN.md' under '# Engineering review: <plan>', and it kept an unfenced copy of the plan's own H1. The named-source route now accepts those spellings and non-inline ledger briefs. The same replay rejects a foreign, mixed, duplicate or missing target, another plan's title or copied H1, a brief naming another plan or file, a mismatched saved brief, and re-asks. The run-36597762183 capture still counts 3.
This commit is contained in:
1 parent
5269452983
commit
8bd53faa6c
3 files changed
+409
-11
No files matched your search
@@ -1,7 +1,11 @@
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import type { NativePlanQuestionCall } from './helpers/plan-count-transcript';
|
||||
import { isEngBatchingIssueAUQ } from './helpers/eng-seeded-coverage';
|
||||
import { nativePlanCallFingerprint } from './helpers/claude-pty-runner';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { createEngBatchingIssueCounter, isEngBatchingIssueAUQ } from './helpers/eng-seeded-coverage';
|
||||
import { engSetupAUQ, hasCompletePlanReport, nativePlanCallFingerprint } from './helpers/claude-pty-runner';
|
||||
import batchingCapture from './fixtures/eng-batching-unsourced-brief-36606688266.json';
|
||||
|
||||
function question(call: NativePlanQuestionCall, text: string) {
|
||||
const answer = call.answers![call.questions[0]!.question]!;
|
||||
@@ -78,3 +82,81 @@ describe('batching caller counts completed issue decisions across setup boundari
|
||||
expect(check(quoted)).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('batching replay of run 36606688266 (unsourced native briefs)', () => {
|
||||
// Run 36606688266 asked one native question per finding (D1-D9 bound to
|
||||
// ledger records R1-R9, D10 a TODO follow-up) but cited no PLAN.md line in the
|
||||
// native brief, so the old detector counted zero review decisions.
|
||||
const FLOOR = 3;
|
||||
const calls = batchingCapture.calls as unknown as NativePlanQuestionCall[];
|
||||
|
||||
function count(plan: string, edit: (calls: NativePlanQuestionCall[]) => void = () => {}) {
|
||||
const copy = structuredClone(calls);
|
||||
edit(copy);
|
||||
const counter = createEngBatchingIssueCounter(() => plan, engSetupAUQ);
|
||||
const counted = copy.filter((call, index) => counter.isReviewAUQ(nativePlanCallFingerprint(call, 0, true), copy.slice(0, index)));
|
||||
return { counted: counted.length, issues: counter.trace.map(entry => entry.issue) };
|
||||
}
|
||||
|
||||
test('the recorded failing verdict is the detector, not the review', () => {
|
||||
expect(batchingCapture.recordedOutcome).toEqual({ outcome: 'completion_summary', step0Count: 10, reviewCount: 0 });
|
||||
expect(calls.every(call => call.answered && call.questions.length === 1)).toBe(true);
|
||||
});
|
||||
|
||||
test('each ledger-bound native decision counts once without a native source citation', () => {
|
||||
const { counted, issues } = count(batchingCapture.plan);
|
||||
expect(issues).toEqual(['R1', 'R2', 'R3', 'R4', 'R5', 'R6', 'R7', 'R8', 'R9'].map(id => `record:${id}`));
|
||||
expect(counted).toBeGreaterThanOrEqual(FLOOR);
|
||||
});
|
||||
|
||||
test('a re-asked decision cannot inflate the count', () => {
|
||||
const { counted } = count(batchingCapture.plan, all => {
|
||||
const again = structuredClone(all[0]!);
|
||||
again.toolUseId += '-again';
|
||||
all.splice(1, 0, again);
|
||||
});
|
||||
expect(counted).toBe(9);
|
||||
});
|
||||
|
||||
const target = 'Review target (fixed): `PLAN.md`';
|
||||
for (const [name, plan] of [
|
||||
['a foreign target', batchingCapture.plan.replace(target, 'Review target (fixed): `OTHER.md`')],
|
||||
['a mixed target', batchingCapture.plan.replace(target, 'Review target (fixed): `OTHER.md` and `PLAN.md`')],
|
||||
['two target declarations', batchingCapture.plan.replace(target, `${target}\nReview target (fixed): \`PLAN.md\``)],
|
||||
['no target declaration', batchingCapture.plan.replace(target, 'Report scope: the fixture repo')],
|
||||
['a report title for another plan', batchingCapture.plan.replace('# Engineering review: Add background job retry framework', '# Engineering review: Replace all customer data')],
|
||||
['an archived report title', batchingCapture.plan.replace('# Engineering review:', '# Archived engineering review:')],
|
||||
['a copied H1 naming another plan', batchingCapture.plan.replace('# Plan: Add background job retry framework', '# Plan: Replace all customer data')],
|
||||
] as const) test(`the unsourced route rejects ${name}`, () => {
|
||||
expect(count(plan).counted).toBe(0);
|
||||
});
|
||||
|
||||
test('the unsourced route rejects a native brief naming another plan or file', () => {
|
||||
const rename = (from: string, to: string) => (all: NativePlanQuestionCall[]) => {
|
||||
for (const call of all) call.questions[0]!.question = call.questions[0]!.question.replace(from, to);
|
||||
};
|
||||
expect(count(batchingCapture.plan, rename('plan "Add background job retry framework"', 'plan "Replace all customer data"')).counted).toBe(0);
|
||||
expect(count(batchingCapture.plan, rename('plan "Add background job retry framework"', 'plan "Add background job retry framework", OTHER.md')).counted).toBe(0);
|
||||
expect(count(batchingCapture.plan, rename('plan "Add background job retry framework"', 'the plan')).counted).toBe(0);
|
||||
});
|
||||
|
||||
test('a saved record whose brief title differs from the native question does not bind it', () => {
|
||||
const plan = batchingCapture.plan.replace(/^Question D1:\n.*$/m, 'Question D1:\nD1 — Some other decision?');
|
||||
expect(count(plan).issues).not.toContain('record:R1');
|
||||
});
|
||||
|
||||
test('the completed report is the early outcome point; a partial report is not', () => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'eng-batching-report-'));
|
||||
try {
|
||||
const report = path.join(dir, 'report.md');
|
||||
fs.writeFileSync(report, batchingCapture.plan);
|
||||
expect(hasCompletePlanReport(report, 0, Date.now() + 1_000)).toBe(true);
|
||||
fs.writeFileSync(report, batchingCapture.plan.slice(0, batchingCapture.plan.indexOf('## Completion summary')));
|
||||
expect(hasCompletePlanReport(report, 0, Date.now() + 1_000)).toBe(false);
|
||||
fs.writeFileSync(report, batchingCapture.plan.replace('## GSTACK REVIEW REPORT', '```\n## GSTACK REVIEW REPORT') + '\n```\n');
|
||||
expect(hasCompletePlanReport(report, 0, Date.now() + 1_000)).toBe(false);
|
||||
} finally {
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user