test(section-loading): credit a Bash print that contains every line of the carved section

This commit is contained in:
garrytan committed 2026-09-30 22:02:15 +00:00
1 parent 5530a7ad1e
commit 005352ddc7
3 files changed
+49 -8

No files matched your search

+27 -7
View File
@@ -209,6 +209,29 @@ export function hasDisabledOutsideReview(output: string): boolean {
return false; return false;
} }
/**
* Sections a capture loaded: a Read of the section file, or a Bash print of it
* (cat/sed ranges, as in run 36776104571) whose outputs together contain every
* line of the section as it stood before the run. A command without that
* printed content, such as head or grep, is not a read.
*/
export function detectSectionReads(toolCalls: SkillTestResult['toolCalls'], sections: Map<string, string>): Set<string> {
const readSections = new Set<string>();
for (const c of toolCalls) {
if (c.tool !== 'Read') continue;
const fp = String(c.input?.file_path ?? '');
const m = fp.match(/(?:^|[\\/])sections[\\/]([A-Za-z0-9._-]+\.md)(?=$|[?#])/);
if (m) readSections.add(m[1]);
}
for (const [name, content] of sections) {
const lines = content.split('\n').map(line => line.trimEnd()).filter(Boolean);
const printed = new Set(toolCalls.filter(c => c.tool === 'Bash' && String(c.input?.command ?? '').includes(`sections/${name}`))
.flatMap(c => c.output.split('\n').map(line => line.trimEnd())));
if (lines.length && lines.every(line => printed.has(line))) readSections.add(name);
}
return readSections;
}
export async function captureSectionReads(opts: { export async function captureSectionReads(opts: {
planDir: string; planDir: string;
skillName: string; skillName: string;
@@ -266,6 +289,9 @@ export async function captureSectionReads(opts: {
}; };
const beforeReport = readReport(); const beforeReport = readReport();
const skillPath = path.join(opts.planDir, opts.skillName, 'SKILL.md'); const skillPath = path.join(opts.planDir, opts.skillName, 'SKILL.md');
const sectionsDir = path.join(opts.planDir, opts.skillName, 'sections');
const sections = new Map(fs.existsSync(sectionsDir) ? fs.readdirSync(sectionsDir)
.filter(name => name.endsWith('.md')).map(name => [name, fs.readFileSync(path.join(sectionsDir, name), 'utf-8')]) : []);
// Outside-review dispatch has separate behavioral coverage. Native-only // Outside-review dispatch has separate behavioral coverage. Native-only
// captures use the real supported control in state owned by this call; // captures use the real supported control in state owned by this call;
// never mutate the operator's or another capture's gstack configuration. // never mutate the operator's or another capture's gstack configuration.
@@ -320,13 +346,7 @@ ${fullPlanReview ? `- Save the evolving plan and review outputs to ${outFile} wi
if (stateDir) fs.rmSync(stateDir, { recursive: true, force: true }); if (stateDir) fs.rmSync(stateDir, { recursive: true, force: true });
} }
const readSections = new Set<string>(); const readSections = detectSectionReads(result.toolCalls, sections);
for (const c of result.toolCalls) {
if (c.tool !== 'Read') continue;
const fp = String(c.input?.file_path ?? '');
const m = fp.match(/(?:^|[\\/])sections[\\/]([A-Za-z0-9._-]+\.md)(?=$|[?#])/);
if (m) readSections.add(m[1]);
}
const afterReport = readReport(); const afterReport = readReport();
const reportWritten = afterReport !== undefined const reportWritten = afterReport !== undefined
+20
View File
@@ -483,3 +483,23 @@ Rules for this run:
} }
}finally{if(child.exitCode===null)child.kill();await child.exited;fs.rmSync(dir,{recursive:true,force:true});} }finally{if(child.exitCode===null)child.kill();await child.exited;fs.rmSync(dir,{recursive:true,force:true});}
},10000); },10000);
// Run 36776104571: the agent printed the whole carved section with Bash sed ranges.
test('section detection credits a complete Bash print of the section, never a partial one', async () => {
const { detectSectionReads } = await import('./helpers/auq-sdk-capture');
const file = path.resolve(import.meta.dir, '..', 'plan-ceo-review/sections/review-sections.md');
const content = fs.readFileSync(file, 'utf-8'), lines = content.split('\n');
const sed = (from: number, to: number, extra = '') => ({ tool: 'Bash',
input: { command: `sed -n ${from},${to}p /fixture/plan-ceo-review/sections/review-sections.md${extra}` },
output: lines.slice(from - 1, to).join('\n') });
const sections = new Map([['review-sections.md', content]]);
const ranges = [[1, 330], [330, 660], [660, 1100], [1100, lines.length]].map(([a, b]) => sed(a!, b!));
const read = (calls: Array<{ tool: string; input: any; output: string }>) => [...detectSectionReads(calls, sections)];
expect(read(ranges)).toEqual(['review-sections.md']);
expect(read([{ tool: 'Bash', input: { command: 'cat sections/review-sections.md' }, output: content }])).toEqual(['review-sections.md']);
expect(read(ranges.filter((_, i) => i !== 2))).toEqual([]);
expect(read([{ ...ranges[0]!, input: { command: 'head -330 PLAN.md' } }, ...ranges.slice(1)])).toEqual([]);
expect(read([{ tool: 'Bash', input: { command: 'cat sections/review-sections.md' }, output: '' }])).toEqual([]);
expect(read(ranges.map(call => ({ ...call, tool: 'Grep' })))).toEqual([]);
expect(read([{ tool: 'Read', input: { file_path: '/fixture/plan-ceo-review/sections/review-sections.md' }, output: '' }])).toEqual(['review-sections.md']);
});
@@ -10,7 +10,8 @@
* loop both defeat the regex, so it reported `read: []` even when the agent did the * loop both defeat the regex, so it reported `read: []` even when the agent did the
* work. It now runs the skill through `claude -p` (the SDK path the AUQ matrix * work. It now runs the skill through `claude -p` (the SDK path the AUQ matrix
* uses) and detects section reads from the tool-use stream (`Read` calls whose * uses) and detects section reads from the tool-use stream (`Read` calls whose
* file_path contains `sections/review-sections.md`). No rendering layer to mangle. * file_path contains `sections/review-sections.md`, or Bash prints whose output
* contains every line of that section). No rendering layer to mangle.
* *
* Hermetic, not install-mutating: the freshly-generated worktree skeleton + * Hermetic, not install-mutating: the freshly-generated worktree skeleton +
* sections are copied into a throwaway fixture dir and the absolute path is pinned, * sections are copied into a throwaway fixture dir and the absolute path is pinned,