From db91607a7fd9419592eb63e036149920304e32b5 Mon Sep 17 00:00:00 2001 From: garrytan Date: Sun, 4 Oct 2026 05:11:28 +0000 Subject: [PATCH] eval(section-loading): byte-range prints of a section count as a complete read Periodic census 37178143007: plan-ceo-section-loading printed review-sections.md in five byte ranges (head -c 16000, then tail -c +N | head -c 17000, one-byte overlaps) and the detector, which matched whole printed lines, saw the lines split at each range edge as missing. Outputs of the commands that name the section are also joined on their overlap. Regression: the captured ranges count; dropping a middle range or the last range does not (pre-fix: the complete ranges were refused). --- test/helpers/auq-sdk-capture.ts | 19 +++++++++++++------ test/section-capture-native-tools.test.ts | 21 +++++++++++++++++++++ 2 files changed, 34 insertions(+), 6 deletions(-) diff --git a/test/helpers/auq-sdk-capture.ts b/test/helpers/auq-sdk-capture.ts index 6fa716d19..765623558 100644 --- a/test/helpers/auq-sdk-capture.ts +++ b/test/helpers/auq-sdk-capture.ts @@ -228,9 +228,11 @@ export function hasDisabledOutsideReview(output: string): boolean { /** * Sections a capture loaded: a Read of the section file, or a Bash print of it - * (cat/sed ranges, as in run 36776104571) whose outputs together contain every - * line of the section as it stood before the run. A command without that - * printed content, such as head or grep, is not a read. + * (cat/sed ranges, as in run 36776104571; byte ranges such as head -c/tail -c, + * as in census 37178143007) whose outputs together contain every line of the + * section as it stood before the run. Byte ranges split lines at their edges, + * so consecutive outputs are also joined on their overlap. A command without + * that printed content, such as head or grep, is not a read. */ export function detectSectionReads(toolCalls: SkillTestResult['toolCalls'], sections: Map): Set { const readSections = new Set(); @@ -242,9 +244,14 @@ export function detectSectionReads(toolCalls: SkillTestResult['toolCalls'], sect } for (const [name, content] of sections) { const lines = content.split('\n').map(line => line.trimEnd()).filter(Boolean); - const printed = new Set(toolCalls.filter(c => c.tool === 'Bash' && String(c.input?.command ?? '').includes(`sections/${name}`)) - .flatMap(c => c.output.split('\n').map(line => line.trimEnd()))); - if (lines.length && lines.every(line => printed.has(line))) readSections.add(name); + const outputs = toolCalls.filter(c => c.tool === 'Bash' && String(c.input?.command ?? '').includes(`sections/${name}`)).map(c => c.output); + const printed = new Set(outputs.flatMap(output => output.split('\n').map(line => line.trimEnd()))); + const joined = outputs.reduce((text, next) => { + let overlap = Math.min(text.length, next.length, 256); + while (overlap > 0 && !text.endsWith(next.slice(0, overlap))) overlap--; + return text + next.slice(overlap); + }, ''); + if (lines.length && lines.every(line => printed.has(line) || joined.includes(line))) readSections.add(name); } return readSections; } diff --git a/test/section-capture-native-tools.test.ts b/test/section-capture-native-tools.test.ts index 1ec7b464b..a6a3b2429 100644 --- a/test/section-capture-native-tools.test.ts +++ b/test/section-capture-native-tools.test.ts @@ -498,3 +498,24 @@ test('section detection credits a complete Bash print of the section, never a pa expect(read(ranges.map(call => ({ ...call, tool: 'Grep' })))).toEqual([]); expect(read([{ tool: 'Read', input: { file_path: '/fixture/plan-ceo-review/sections/review-sections.md' }, output: '' }])).toEqual(['review-sections.md']); }); + +// Census 37178143007: the agent printed the section in byte ranges that split +// lines at their edges (head -c, then tail -c +N | head -c, one-byte overlaps). +test('section detection credits complete byte-range prints and still refuses a gap', async () => { + const { detectSectionReads } = await import('./helpers/auq-sdk-capture'); + const file = path.resolve(import.meta.dir, '..', 'plan-ceo-review/sections/review-sections.md'); + const content = fs.readFileSync(file, 'utf-8'), bytes = Buffer.from(content); + const sections = new Map([['review-sections.md', content]]); + const chunk = (command: string, from: number, length?: number) => ({ tool: 'Bash', input: { command }, + output: bytes.subarray(from, length === undefined ? undefined : from + length).toString('utf8') }); + const calls = [ + chunk('wc -c plan-ceo-review/sections/review-sections.md && head -c 16000 plan-ceo-review/sections/review-sections.md', 0, 16000), + ...[16000, 33000, 50000].map(start => chunk(`tail -c +${start} plan-ceo-review/sections/review-sections.md | head -c 17000`, start - 1, 17000)), + chunk('tail -c +67000 plan-ceo-review/sections/review-sections.md', 66999), + ]; + const read = (list: typeof calls) => [...detectSectionReads(list, sections)]; + expect(bytes.length).toBeGreaterThan(67000); + expect(read(calls)).toEqual(['review-sections.md']); + expect(read(calls.filter((_, i) => i !== 2))).toEqual([]); + expect(read(calls.slice(0, -1))).toEqual([]); +});