mirror of
https://github.com/garrytan/gstack.git
synced 2026-10-03 09:56:57 +02:00
test(section-loading): credit a Bash print that contains every line of the carved section
This commit is contained in:
1 parent
5530a7ad1e
commit
005352ddc7
3 files changed
+49
-8
No files matched your search
@@ -209,6 +209,29 @@ export function hasDisabledOutsideReview(output: string): boolean {
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Sections a capture loaded: a Read of the section file, or a Bash print of it
|
||||||
|
* (cat/sed ranges, as in run 36776104571) whose outputs together contain every
|
||||||
|
* line of the section as it stood before the run. A command without that
|
||||||
|
* printed content, such as head or grep, is not a read.
|
||||||
|
*/
|
||||||
|
export function detectSectionReads(toolCalls: SkillTestResult['toolCalls'], sections: Map<string, string>): Set<string> {
|
||||||
|
const readSections = new Set<string>();
|
||||||
|
for (const c of toolCalls) {
|
||||||
|
if (c.tool !== 'Read') continue;
|
||||||
|
const fp = String(c.input?.file_path ?? '');
|
||||||
|
const m = fp.match(/(?:^|[\\/])sections[\\/]([A-Za-z0-9._-]+\.md)(?=$|[?#])/);
|
||||||
|
if (m) readSections.add(m[1]);
|
||||||
|
}
|
||||||
|
for (const [name, content] of sections) {
|
||||||
|
const lines = content.split('\n').map(line => line.trimEnd()).filter(Boolean);
|
||||||
|
const printed = new Set(toolCalls.filter(c => c.tool === 'Bash' && String(c.input?.command ?? '').includes(`sections/${name}`))
|
||||||
|
.flatMap(c => c.output.split('\n').map(line => line.trimEnd())));
|
||||||
|
if (lines.length && lines.every(line => printed.has(line))) readSections.add(name);
|
||||||
|
}
|
||||||
|
return readSections;
|
||||||
|
}
|
||||||
|
|
||||||
export async function captureSectionReads(opts: {
|
export async function captureSectionReads(opts: {
|
||||||
planDir: string;
|
planDir: string;
|
||||||
skillName: string;
|
skillName: string;
|
||||||
@@ -266,6 +289,9 @@ export async function captureSectionReads(opts: {
|
|||||||
};
|
};
|
||||||
const beforeReport = readReport();
|
const beforeReport = readReport();
|
||||||
const skillPath = path.join(opts.planDir, opts.skillName, 'SKILL.md');
|
const skillPath = path.join(opts.planDir, opts.skillName, 'SKILL.md');
|
||||||
|
const sectionsDir = path.join(opts.planDir, opts.skillName, 'sections');
|
||||||
|
const sections = new Map(fs.existsSync(sectionsDir) ? fs.readdirSync(sectionsDir)
|
||||||
|
.filter(name => name.endsWith('.md')).map(name => [name, fs.readFileSync(path.join(sectionsDir, name), 'utf-8')]) : []);
|
||||||
// Outside-review dispatch has separate behavioral coverage. Native-only
|
// Outside-review dispatch has separate behavioral coverage. Native-only
|
||||||
// captures use the real supported control in state owned by this call;
|
// captures use the real supported control in state owned by this call;
|
||||||
// never mutate the operator's or another capture's gstack configuration.
|
// never mutate the operator's or another capture's gstack configuration.
|
||||||
@@ -320,13 +346,7 @@ ${fullPlanReview ? `- Save the evolving plan and review outputs to ${outFile} wi
|
|||||||
if (stateDir) fs.rmSync(stateDir, { recursive: true, force: true });
|
if (stateDir) fs.rmSync(stateDir, { recursive: true, force: true });
|
||||||
}
|
}
|
||||||
|
|
||||||
const readSections = new Set<string>();
|
const readSections = detectSectionReads(result.toolCalls, sections);
|
||||||
for (const c of result.toolCalls) {
|
|
||||||
if (c.tool !== 'Read') continue;
|
|
||||||
const fp = String(c.input?.file_path ?? '');
|
|
||||||
const m = fp.match(/(?:^|[\\/])sections[\\/]([A-Za-z0-9._-]+\.md)(?=$|[?#])/);
|
|
||||||
if (m) readSections.add(m[1]);
|
|
||||||
}
|
|
||||||
|
|
||||||
const afterReport = readReport();
|
const afterReport = readReport();
|
||||||
const reportWritten = afterReport !== undefined
|
const reportWritten = afterReport !== undefined
|
||||||
|
|||||||
@@ -483,3 +483,23 @@ Rules for this run:
|
|||||||
}
|
}
|
||||||
}finally{if(child.exitCode===null)child.kill();await child.exited;fs.rmSync(dir,{recursive:true,force:true});}
|
}finally{if(child.exitCode===null)child.kill();await child.exited;fs.rmSync(dir,{recursive:true,force:true});}
|
||||||
},10000);
|
},10000);
|
||||||
|
|
||||||
|
// Run 36776104571: the agent printed the whole carved section with Bash sed ranges.
|
||||||
|
test('section detection credits a complete Bash print of the section, never a partial one', async () => {
|
||||||
|
const { detectSectionReads } = await import('./helpers/auq-sdk-capture');
|
||||||
|
const file = path.resolve(import.meta.dir, '..', 'plan-ceo-review/sections/review-sections.md');
|
||||||
|
const content = fs.readFileSync(file, 'utf-8'), lines = content.split('\n');
|
||||||
|
const sed = (from: number, to: number, extra = '') => ({ tool: 'Bash',
|
||||||
|
input: { command: `sed -n ${from},${to}p /fixture/plan-ceo-review/sections/review-sections.md${extra}` },
|
||||||
|
output: lines.slice(from - 1, to).join('\n') });
|
||||||
|
const sections = new Map([['review-sections.md', content]]);
|
||||||
|
const ranges = [[1, 330], [330, 660], [660, 1100], [1100, lines.length]].map(([a, b]) => sed(a!, b!));
|
||||||
|
const read = (calls: Array<{ tool: string; input: any; output: string }>) => [...detectSectionReads(calls, sections)];
|
||||||
|
expect(read(ranges)).toEqual(['review-sections.md']);
|
||||||
|
expect(read([{ tool: 'Bash', input: { command: 'cat sections/review-sections.md' }, output: content }])).toEqual(['review-sections.md']);
|
||||||
|
expect(read(ranges.filter((_, i) => i !== 2))).toEqual([]);
|
||||||
|
expect(read([{ ...ranges[0]!, input: { command: 'head -330 PLAN.md' } }, ...ranges.slice(1)])).toEqual([]);
|
||||||
|
expect(read([{ tool: 'Bash', input: { command: 'cat sections/review-sections.md' }, output: '' }])).toEqual([]);
|
||||||
|
expect(read(ranges.map(call => ({ ...call, tool: 'Grep' })))).toEqual([]);
|
||||||
|
expect(read([{ tool: 'Read', input: { file_path: '/fixture/plan-ceo-review/sections/review-sections.md' }, output: '' }])).toEqual(['review-sections.md']);
|
||||||
|
});
|
||||||
@@ -10,7 +10,8 @@
|
|||||||
* loop both defeat the regex, so it reported `read: []` even when the agent did the
|
* loop both defeat the regex, so it reported `read: []` even when the agent did the
|
||||||
* work. It now runs the skill through `claude -p` (the SDK path the AUQ matrix
|
* work. It now runs the skill through `claude -p` (the SDK path the AUQ matrix
|
||||||
* uses) and detects section reads from the tool-use stream (`Read` calls whose
|
* uses) and detects section reads from the tool-use stream (`Read` calls whose
|
||||||
* file_path contains `sections/review-sections.md`). No rendering layer to mangle.
|
* file_path contains `sections/review-sections.md`, or Bash prints whose output
|
||||||
|
* contains every line of that section). No rendering layer to mangle.
|
||||||
*
|
*
|
||||||
* Hermetic, not install-mutating: the freshly-generated worktree skeleton +
|
* Hermetic, not install-mutating: the freshly-generated worktree skeleton +
|
||||||
* sections are copied into a throwaway fixture dir and the absolute path is pinned,
|
* sections are copied into a throwaway fixture dir and the absolute path is pinned,
|
||||||
|
|||||||
Reference in new issue
Block a user