From 005352ddc7cb03dd7da89ee85d4f86eea236eb25 Mon Sep 17 00:00:00 2001 From: garrytan Date: Wed, 30 Sep 2026 21:51:52 +0000 Subject: [PATCH] test(section-loading): credit a Bash print that contains every line of the carved section --- test/helpers/auq-sdk-capture.ts | 34 +++++++++++++++---- test/section-capture-native-tools.test.ts | 20 +++++++++++ ...2e-plan-ceo-review-section-loading.test.ts | 3 +- 3 files changed, 49 insertions(+), 8 deletions(-) diff --git a/test/helpers/auq-sdk-capture.ts b/test/helpers/auq-sdk-capture.ts index 5f64f4846..46c4d5c94 100644 --- a/test/helpers/auq-sdk-capture.ts +++ b/test/helpers/auq-sdk-capture.ts @@ -209,6 +209,29 @@ export function hasDisabledOutsideReview(output: string): boolean { return false; } +/** + * Sections a capture loaded: a Read of the section file, or a Bash print of it + * (cat/sed ranges, as in run 36776104571) whose outputs together contain every + * line of the section as it stood before the run. A command without that + * printed content, such as head or grep, is not a read. + */ +export function detectSectionReads(toolCalls: SkillTestResult['toolCalls'], sections: Map): Set { + const readSections = new Set(); + for (const c of toolCalls) { + if (c.tool !== 'Read') continue; + const fp = String(c.input?.file_path ?? ''); + const m = fp.match(/(?:^|[\\/])sections[\\/]([A-Za-z0-9._-]+\.md)(?=$|[?#])/); + if (m) readSections.add(m[1]); + } + for (const [name, content] of sections) { + const lines = content.split('\n').map(line => line.trimEnd()).filter(Boolean); + const printed = new Set(toolCalls.filter(c => c.tool === 'Bash' && String(c.input?.command ?? '').includes(`sections/${name}`)) + .flatMap(c => c.output.split('\n').map(line => line.trimEnd()))); + if (lines.length && lines.every(line => printed.has(line))) readSections.add(name); + } + return readSections; +} + export async function captureSectionReads(opts: { planDir: string; skillName: string; @@ -266,6 +289,9 @@ export async function captureSectionReads(opts: { }; const beforeReport = readReport(); const skillPath = path.join(opts.planDir, opts.skillName, 'SKILL.md'); + const sectionsDir = path.join(opts.planDir, opts.skillName, 'sections'); + const sections = new Map(fs.existsSync(sectionsDir) ? fs.readdirSync(sectionsDir) + .filter(name => name.endsWith('.md')).map(name => [name, fs.readFileSync(path.join(sectionsDir, name), 'utf-8')]) : []); // Outside-review dispatch has separate behavioral coverage. Native-only // captures use the real supported control in state owned by this call; // never mutate the operator's or another capture's gstack configuration. @@ -320,13 +346,7 @@ ${fullPlanReview ? `- Save the evolving plan and review outputs to ${outFile} wi if (stateDir) fs.rmSync(stateDir, { recursive: true, force: true }); } - const readSections = new Set(); - for (const c of result.toolCalls) { - if (c.tool !== 'Read') continue; - const fp = String(c.input?.file_path ?? ''); - const m = fp.match(/(?:^|[\\/])sections[\\/]([A-Za-z0-9._-]+\.md)(?=$|[?#])/); - if (m) readSections.add(m[1]); - } + const readSections = detectSectionReads(result.toolCalls, sections); const afterReport = readReport(); const reportWritten = afterReport !== undefined diff --git a/test/section-capture-native-tools.test.ts b/test/section-capture-native-tools.test.ts index 36f86400f..8f3e48c0e 100644 --- a/test/section-capture-native-tools.test.ts +++ b/test/section-capture-native-tools.test.ts @@ -483,3 +483,23 @@ Rules for this run: } }finally{if(child.exitCode===null)child.kill();await child.exited;fs.rmSync(dir,{recursive:true,force:true});} },10000); + +// Run 36776104571: the agent printed the whole carved section with Bash sed ranges. +test('section detection credits a complete Bash print of the section, never a partial one', async () => { + const { detectSectionReads } = await import('./helpers/auq-sdk-capture'); + const file = path.resolve(import.meta.dir, '..', 'plan-ceo-review/sections/review-sections.md'); + const content = fs.readFileSync(file, 'utf-8'), lines = content.split('\n'); + const sed = (from: number, to: number, extra = '') => ({ tool: 'Bash', + input: { command: `sed -n ${from},${to}p /fixture/plan-ceo-review/sections/review-sections.md${extra}` }, + output: lines.slice(from - 1, to).join('\n') }); + const sections = new Map([['review-sections.md', content]]); + const ranges = [[1, 330], [330, 660], [660, 1100], [1100, lines.length]].map(([a, b]) => sed(a!, b!)); + const read = (calls: Array<{ tool: string; input: any; output: string }>) => [...detectSectionReads(calls, sections)]; + expect(read(ranges)).toEqual(['review-sections.md']); + expect(read([{ tool: 'Bash', input: { command: 'cat sections/review-sections.md' }, output: content }])).toEqual(['review-sections.md']); + expect(read(ranges.filter((_, i) => i !== 2))).toEqual([]); + expect(read([{ ...ranges[0]!, input: { command: 'head -330 PLAN.md' } }, ...ranges.slice(1)])).toEqual([]); + expect(read([{ tool: 'Bash', input: { command: 'cat sections/review-sections.md' }, output: '' }])).toEqual([]); + expect(read(ranges.map(call => ({ ...call, tool: 'Grep' })))).toEqual([]); + expect(read([{ tool: 'Read', input: { file_path: '/fixture/plan-ceo-review/sections/review-sections.md' }, output: '' }])).toEqual(['review-sections.md']); +}); diff --git a/test/skill-e2e-plan-ceo-review-section-loading.test.ts b/test/skill-e2e-plan-ceo-review-section-loading.test.ts index 582b40667..7af32cccd 100644 --- a/test/skill-e2e-plan-ceo-review-section-loading.test.ts +++ b/test/skill-e2e-plan-ceo-review-section-loading.test.ts @@ -10,7 +10,8 @@ * loop both defeat the regex, so it reported `read: []` even when the agent did the * work. It now runs the skill through `claude -p` (the SDK path the AUQ matrix * uses) and detects section reads from the tool-use stream (`Read` calls whose - * file_path contains `sections/review-sections.md`). No rendering layer to mangle. + * file_path contains `sections/review-sections.md`, or Bash prints whose output + * contains every line of that section). No rendering layer to mangle. * * Hermetic, not install-mutating: the freshly-generated worktree skeleton + * sections are copied into a throwaway fixture dir and the absolute path is pinned,