Files
gstack/test/plan-create-prepublication.test.ts
Garry Tanandcapy-ai[bot] 6f7488a70f v1.91.185.0 fix(evals): plan-design focus rating in plain words, docsync own-call rule where --help is named, floor Create crop and 0D menu (GSTA-65) (#3358)
* fix(evals): plan-design focus rating in plain words, docsync own-call rule at the candidate --help, floor Create menu cropped at its rule (GSTA-65)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* fix(evals): floor actor answers Step 0D's focus menu in the template's own words (GSTA-65)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* fix(evals): floor actor identifies Step 0D's focus menu by its structure, not one wording (GSTA-65)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* docs(changelog): GSTA-65 paid-side flaky repairs under [Unreleased]

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* fix(evals): the 0D focus picker moves into plan-design-finding-floor's owned fixture; no harness series reset (GSTA-65)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* test(evals): the 0D picker rows join the floor case's owner test (test-of-test ratchet)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* v1.91.185.0 fix(evals): plan-design focus rating in plain words, docsync own-call rule, floor Create crop and 0D menu (GSTA-65) (#3358)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

---------

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
2026-10-23 18:35:59 +00:00

285 lines
20 KiB
TypeScript

import {expect,test} from 'bun:test';
import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import {createHash} from 'node:crypto';
import {createFilePermissionRecorder,recordFilePermission,currentFilePermissionEpoch,readPendingWriteInput} from './helpers/plan-count-file-permission';
import {readPlanCountTranscript} from './helpers/plan-count-transcript';
import {createPlanCountPermissionGuard,countingCapture,type CountingRun} from './helpers/claude-pty-runner';
import {createPlanCountSnapshotWriter} from './helpers/plan-count-artifacts';
import captured from './fixtures/plan-create-prepublication-491.json';
import toolOnlyCreate from './fixtures/plan-ceo-floor-toolonly-create.json';
import quoteWrapCreate from './fixtures/plan-ceo-floor-create-quote-wrap.json';
import ruleCropCreate from './fixtures/plan-design-floor-create-rule-crop.json';
function fixture(){
const root=fs.mkdtempSync(path.join(os.tmpdir(),'gstack-prewrite-'));const cwd=path.join(root,'owned'),config=path.join(root,'config');fs.mkdirSync(cwd);
const expected=path.join(cwd,path.basename(captured.originalHook.expected)),sid=captured.originalHook.sessionId;
const id=captured.originalHook.pendingId.slice(sid.length+1),journal=path.join(config,'projects','owned',sid+'.jsonl');fs.mkdirSync(path.dirname(journal),{recursive:true});
const startedAt=Date.now()-1000;const recorder=createFilePermissionRecorder(cwd,config,expected)!;
const input={file_path:expected,content:captured.controlledWriteContent};
const native=(role:string,content:any[])=>({cwd,sessionId:sid,isSidechain:false,timestamp:new Date().toISOString(),message:{role,content}});
const base=[native('assistant',[{type:'text',text:'Preparing the requested review.'}])];
const write=(rows:any[])=>fs.writeFileSync(journal,rows.map(r=>JSON.stringify(r)).join('\n')+'\n');write(base);
const hook=(kind='PreToolUse',toolId=id,toolInput:any=input,extra={})=>recordFilePermission(JSON.stringify({hook_event_name:kind,tool_name:'Write',session_id:sid,tool_use_id:toolId,cwd,transcript_path:journal,tool_input:toolInput,...extra}),recorder.file,cwd,config,expected);
const screen=captured.originalScreen.replaceAll(path.dirname(captured.originalHook.expected),cwd);
const read=()=>currentFilePermissionEpoch(recorder.file,expected,cwd,config,startedAt,readPlanCountTranscript(config,cwd),screen);
const block=(toolId=id,toolInput:any=input,name='Write')=>({type:'tool_use',id:toolId,name,input:toolInput});
const sidecar=recorder.file+'.write.json';
return {root,cwd,config,expected,sid,id,journal,recorder,input,native,base,write,hook,screen,read,block,sidecar,startedAt,close(){recorder.dispose();fs.rmSync(root,{recursive:true,force:true});}};
}
test('owned current Write prepublication input authenticates the captured cropped Create once',()=>{
const f=fixture();try{
f.hook();expect(f.read()?.pendingId).toBe(`${f.sid}:${f.id}`);
const guard=createPlanCountPermissionGuard();expect(guard(f.screen,'',f.read())).toBe('grant');expect(guard(f.screen,'',f.read())).toBe('handled');
const state=JSON.parse(fs.readFileSync(f.recorder.file,'utf8')),bytes=fs.readFileSync(f.sidecar);
expect(state.writeInputSha256).toBe(createHash('sha256').update(bytes).digest('hex'));
expect(JSON.parse(bytes.toString()).input).toEqual(f.input);
expect(fs.readFileSync(f.recorder.file,'utf8')).not.toContain(f.input.content);
expect(fs.statSync(f.sidecar).mode&0o777).toBe(0o600);
}finally{f.close();}
});
test('original content-free capture alone supplies no missing argument or permission credit',()=>{
const f=fixture();try{f.hook();const state=JSON.parse(fs.readFileSync(f.recorder.file,'utf8'));delete state.writeInputSha256;fs.writeFileSync(f.recorder.file,JSON.stringify(state));fs.rmSync(f.sidecar,{force:true});expect(f.read()).toBeNull();}finally{f.close();}
});
test.each(['other-pending','completed','changed-hidden-input','different-tool','conflicting-duplicate'])(`prepublication cannot override %s public native evidence`,mode=>{
const f=fixture();try{f.hook();let rows:any[]=[...f.base];
if(mode==='other-pending')rows.push(f.native('assistant',[f.block('different')]));
if(mode==='completed')rows.push(f.native('user',[{type:'tool_result',tool_use_id:f.id,content:'done'}]));
if(mode==='changed-hidden-input')rows.push(f.native('assistant',[f.block(f.id,{...f.input,content:f.input.content.replace(/^.*$/m,'Changed invisible first line')})]));
if(mode==='different-tool')rows.push(f.native('assistant',[f.block(f.id,f.input,'Read')]));
if(mode==='conflicting-duplicate')rows.push(f.native('assistant',[f.block(),f.block(f.id,{...f.input,content:'different'})]));
f.write(rows);expect(f.read()).toBeNull();
}finally{f.close();}
});
test('matching published Write retains the existing route and exact input agreement',()=>{
const f=fixture();try{f.hook();f.write([...f.base,f.native('assistant',[f.block()])]);expect(f.read()?.pendingId).toBe(`${f.sid}:${f.id}`);}finally{f.close();}
});
test.each(['missing','changed-bytes','linked','wrong-current-id','foreign-target'])(`invalid %s witness fails closed`,mode=>{
const f=fixture();try{f.hook();
if(mode==='missing')fs.unlinkSync(f.sidecar);
if(mode==='changed-bytes')fs.appendFileSync(f.sidecar,' ');
if(mode==='linked'){const copy=f.sidecar+'.copy';fs.copyFileSync(f.sidecar,copy);fs.unlinkSync(f.sidecar);fs.symlinkSync(copy,f.sidecar);}
if(mode==='wrong-current-id'||mode==='foreign-target'){
const witness=JSON.parse(fs.readFileSync(f.sidecar,'utf8'));
if(mode==='wrong-current-id')witness.pendingId=`${f.sid}:different`;else witness.input.file_path=f.expected+'-foreign';
const bytes=JSON.stringify(witness);fs.writeFileSync(f.sidecar,bytes);const state=JSON.parse(fs.readFileSync(f.recorder.file,'utf8'));state.writeInputSha256=createHash('sha256').update(bytes).digest('hex');fs.writeFileSync(f.recorder.file,JSON.stringify(state));
}expect(f.read()).toBeNull();
}finally{f.close();}
});
test('completion and replay cannot retain or reopen a Write witness; a later failed-save recovery is fresh',()=>{
const f=fixture();try{f.hook();f.hook('PostToolUseFailure');expect(f.read()).toBeNull();expect(fs.existsSync(f.sidecar)).toBe(false);f.hook();expect(f.read()).toBeNull();f.hook('PreToolUse','recovery');expect(f.read()?.pendingId).toBe(`${f.sid}:recovery`);f.hook('PostToolUse','recovery');expect(fs.existsSync(f.sidecar)).toBe(false);f.hook('PreToolUse','recovery');expect(f.read()).toBeNull();}finally{f.close();}
});
test('overlapping uncompleted hook epochs never borrow the most recently captured Write',()=>{
const f=fixture();try{f.hook();f.hook('PreToolUse','second');expect(f.read()).toBeNull();f.hook('PostToolUse','second');f.hook('PreToolUse','third');expect(f.read()).toBeNull();}finally{f.close();}
});
test('conflicting repeated current hook arguments invalidate the witness without reopening its ID',()=>{
const f=fixture();try{
f.hook();f.hook();expect(f.read()?.pendingId).toBe(`${f.sid}:${f.id}`);
f.hook('PreToolUse',f.id,{...f.input,content:f.input.content+'\nchanged hidden tail'});
expect(f.read()).toBeNull();
f.hook();expect(f.read()).toBeNull();
f.hook('PostToolUseFailure');f.hook();expect(f.read()).toBeNull();
f.hook('PreToolUse','fresh');expect(f.read()?.pendingId).toBe(`${f.sid}:fresh`);
}finally{f.close();}
});
test.each(['cwd','target','config','stale','future','linked-state','oversized-state','oversized-witness'])(`standalone witness reader rejects %s ownership or bounds`,mode=>{
const f=fixture();try{
f.hook();let cwd=f.cwd,expected=f.expected,config:string|null=f.config,startedAt=f.startedAt;
if(mode==='cwd')cwd+='-foreign';
if(mode==='target')expected+='-foreign';
if(mode==='config')config+='-foreign';
if(mode==='stale')startedAt=Date.now()+1000;
if(mode==='future'){
const state=JSON.parse(fs.readFileSync(f.recorder.file,'utf8'));
state.timestamp=new Date(Date.now()+60_000).toISOString();fs.writeFileSync(f.recorder.file,JSON.stringify(state));
}
if(mode==='linked-state'){
fs.renameSync(f.recorder.file,f.recorder.file+'.copy');fs.symlinkSync(f.recorder.file+'.copy',f.recorder.file);
}
if(mode==='oversized-state')fs.truncateSync(f.recorder.file,64*1024+1);
if(mode==='oversized-witness')fs.truncateSync(f.sidecar,4*1024*1024+1);
expect(readPendingWriteInput(f.recorder.file,expected,cwd,config,startedAt)).toBeUndefined();
}finally{f.close();}
});
test.each(['foreign-session','sidechain','foreign-target','foreign-journal'])(`foreign %s hook cannot supply the cropped Create input`,mode=>{
const f=fixture();try{
const extra:any={};let input:any=f.input;
if(mode==='foreign-session')extra.session_id='foreign';
if(mode==='sidechain')extra.agent_id='foreign';
if(mode==='foreign-target')input={...input,file_path:f.expected+'-foreign'};
if(mode==='foreign-journal')extra.transcript_path=path.join(f.root,'foreign.jsonl');
f.hook('PreToolUse',f.id,input,extra);expect(f.read()).toBeNull();expect(fs.existsSync(f.sidecar)).toBe(false);
}finally{f.close();}
});
test('oversized actual hook input fails closed without a witness',()=>{
const f=fixture();try{
f.hook('PreToolUse',f.id,{...f.input,content:'x'.repeat(4*1024*1024)});
expect(f.read()).toBeNull();expect(fs.existsSync(f.sidecar)).toBe(false);
}finally{f.close();}
});
test('legacy published Write metadata remains sufficient without a sidecar',()=>{
const f=fixture();try{
f.hook();const state=JSON.parse(fs.readFileSync(f.recorder.file,'utf8'));
delete state.writeInputSha256;fs.writeFileSync(f.recorder.file,JSON.stringify(state));fs.unlinkSync(f.sidecar);
f.write([...f.base,f.native('assistant',[f.block()])]);expect(f.read()?.pendingId).toBe(`${f.sid}:${f.id}`);
}finally{f.close();}
});
// Execute only the actual count capture assembly with its real reader/writer.
// This calls the counting runner's capture step directly, not a full loop; existing count throw/timeout tests
// separately prove that the lifecycle invokes capture before fixture disposal.
test('count capture assembly retains exact prepublication input through refresh, throw and fixture cleanup',()=>{
const f=fixture(),evalDir=fs.mkdtempSync(path.join(os.tmpdir(),'count-write-retention-'));
try {
f.hook();const expected=readPendingWriteInput(f.recorder.file,f.expected,f.cwd,f.config,f.startedAt);
expect(expected).toBeDefined();
// The runner's own capture step, called with the state it builds for an attempt.
const run={opts:{skillName:'plan-eng-review'},fixture:{cwd:f.cwd},startedAt:f.startedAt,viewport:f.screen,
ownedFilePermissions:[{file:f.recorder.file,expected:f.expected}],
saveSnapshot:createPlanCountSnapshotWriter({EVALS_RUN_ID:'count-prepublication-free',GSTACK_EVAL_DIR:evalDir})} as unknown as CountingRun;
const capture=(observation:object)=>countingCapture(run,{hermeticConfigDir:f.config,rawOutput:()=>f.screen,visibleText:()=>f.screen},observation);
const progress=capture({state:'in_progress'});
expect(progress.artifactError).toBeUndefined();expect(progress.artifactDir).toBeDefined();
const initial=JSON.parse(fs.readFileSync(path.join(progress.artifactDir,'observation.json'),'utf8'));
expect(initial.state).toBe('in_progress');expect(initial.pendingWriteInputs).toEqual([expected]);expect(initial.publicTools).toEqual([]);
const published=f.native('assistant',[f.block()]);f.write([...f.base,published]);
const thrown=capture({state:'threw',error:'controlled caller interruption'});
expect(thrown.artifactDir).toBe(progress.artifactDir);expect(thrown.artifactError).toBeUndefined();
f.close();
const saved=JSON.parse(fs.readFileSync(path.join(thrown.artifactDir,'observation.json'),'utf8'));
expect(saved.state).toBe('threw');expect(saved.error).toBe('controlled caller interruption');
expect(saved.pendingWriteInputs).toEqual([expected]);expect(fs.existsSync(f.recorder.file)).toBe(false);
expect(saved.publicTools).toEqual([{sessionId:f.sid,timestamp:published.timestamp,toolUseId:f.id,kind:'use',name:'Write',input:f.input}]);
expect(fs.existsSync(f.sidecar)).toBe(false);expect(fs.existsSync(f.journal)).toBe(false);
expect(fs.readFileSync(path.join(thrown.artifactDir,'terminal.screen.log'),'utf8')).toBe(f.screen);
expect(fs.statSync(path.join(thrown.artifactDir,'observation.json')).mode&0o777).toBe(0o600);
}finally{f.close();fs.rmSync(evalDir,{recursive:true,force:true});}
});
// CI 39161502626 slice 6: the parent ran four tools, then its first Write asked to
// create the owned working plan before any assistant text or question existed.
function toolOnlyFixture(){
const root=fs.mkdtempSync(path.join(os.tmpdir(),'gstack-toolonly-'));const cwd=path.join(root,'owned'),config=path.join(root,'config');fs.mkdirSync(cwd);
const expected=path.join(cwd,toolOnlyCreate.basename),sid=toolOnlyCreate.sessionId,id=toolOnlyCreate.pendingId.slice(sid.length+1);
const journal=path.join(config,'projects','owned',sid+'.jsonl');fs.mkdirSync(path.dirname(journal),{recursive:true});
const startedAt=Date.now()-1000;const recorder=createFilePermissionRecorder(cwd,config,expected)!;
const native=(role:string,content:any[],session=sid)=>({cwd,sessionId:session,isSidechain:false,timestamp:new Date().toISOString(),message:{role,content}});
const tools=(session=sid)=>toolOnlyCreate.publicTools.map(event=>event.kind==='use'
? native('assistant',[{type:'tool_use',id:event.toolUseId,name:event.name,input:event.name==='Read'?{file_path:path.join(cwd,'PLAN.md')}:{command:'true'}}],session)
: native('user',[{type:'tool_result',tool_use_id:event.toolUseId,content:'ok'}],session));
const write=(rows:any[],file=journal)=>{fs.mkdirSync(path.dirname(file),{recursive:true});fs.writeFileSync(file,rows.map(r=>JSON.stringify(r)).join('\n')+'\n');};
write(tools());
recordFilePermission(JSON.stringify({hook_event_name:'PreToolUse',tool_name:'Write',session_id:sid,tool_use_id:id,cwd,transcript_path:journal,
tool_input:{file_path:expected,content:toolOnlyCreate.writeContent}}),recorder.file,cwd,config,expected);
const read=(screen=toolOnlyCreate.screen)=>currentFilePermissionEpoch(recorder.file,expected,cwd,config,startedAt,readPlanCountTranscript(config,cwd),screen);
return {root,cwd,config,sid,id,journal,native,tools,write,read,close(){recorder.dispose();fs.rmSync(root,{recursive:true,force:true});}};
}
test('a tool-only parent session authenticates its captured owned Create once',()=>{
const f=toolOnlyFixture();try{
expect(toolOnlyCreate.recorded).toMatchObject({outcome:'timeout',transcriptCalls:0,transcriptAssistantMessages:0});
const transcript=readPlanCountTranscript(f.config,f.cwd);
expect(transcript).toMatchObject({status:'ready',calls:[],assistantMessages:[]});
expect(f.read()?.pendingId).toBe(toolOnlyCreate.pendingId);
const guard=createPlanCountPermissionGuard();
expect(guard(toolOnlyCreate.screen,'',f.read())).toBe('grant');
expect(guard(toolOnlyCreate.screen,'',f.read())).toBe('handled');
}finally{f.close();}
});
test('tool-only identity still requires exactly one parent session that owns the pending Write',()=>{
const f=toolOnlyFixture();try{
const foreign='11111111-2222-4333-8444-555555555555',foreignJournal=path.join(f.config,'projects','owned',foreign+'.jsonl');
f.write(f.tools(foreign),foreignJournal);
expect(f.read()).toBeNull();
f.write([]);
expect(f.read()).toBeNull();
fs.rmSync(foreignJournal);
expect(f.read()).toBeNull();
f.write([...f.tools(),f.native('assistant',[{type:'tool_use',id:'other-write',name:'Write',input:{file_path:path.join(f.cwd,'other.md'),content:'x'}}])]);
expect(f.read()).toBeNull();
}finally{f.close();}
});
test('a tool-only session gets no credit for a changed preview or a foreign create prompt',()=>{
const f=toolOnlyFixture();try{
expect(f.read(toolOnlyCreate.screen.replace('Validate first (recommended)','Validate later (recommended)'))).toBeNull();
expect(f.read(toolOnlyCreate.screen.replaceAll(toolOnlyCreate.basename,'foreign-plan.md'))).toBeUndefined();
}finally{f.close();}
});
// Focused trial t3 (2026-10-20): a soft-wrapped table row continued with '>activation->paid)' and the
// blockquote guard refused the owned Create menu until permission_unanswered.
type CapturedCreate={basename:string;sessionId:string;toolUseId:string;writeContent:string;screen:string};
function quoteWrap(capture:CapturedCreate=quoteWrapCreate){
const root=fs.mkdtempSync(path.join(os.tmpdir(),'gstack-quote-wrap-')),cwd=path.join(root,'owned'),config=path.join(root,'config');fs.mkdirSync(cwd);
const expected=path.join(cwd,capture.basename),sid=capture.sessionId,id=capture.toolUseId;
const journal=path.join(config,'projects','owned',sid+'.jsonl');fs.mkdirSync(path.dirname(journal),{recursive:true});
const startedAt=Date.now()-1000,recorder=createFilePermissionRecorder(cwd,config,expected)!,input={file_path:expected,content:capture.writeContent};
fs.writeFileSync(journal,JSON.stringify({cwd,sessionId:sid,isSidechain:false,timestamp:new Date().toISOString(),message:{role:'assistant',content:[{type:'text',text:'Writing the working plan.'}]}})+'\n');
recordFilePermission(JSON.stringify({hook_event_name:'PreToolUse',tool_name:'Write',session_id:sid,tool_use_id:id,cwd,transcript_path:journal,tool_input:input}),recorder.file,cwd,config,expected);
const read=(screen=capture.screen)=>currentFilePermissionEpoch(recorder.file,expected,cwd,config,startedAt,readPlanCountTranscript(config,cwd),screen);
return {sid,id,read,close(){recorder.dispose();fs.rmSync(root,{recursive:true,force:true});}};
}
test('t3: a wrapped preview row that continues with ">" still binds the owned Create once',()=>{
const f=quoteWrap();try{
expect(createHash('sha256').update(quoteWrapCreate.screen).digest('hex')).toBe(quoteWrapCreate.screenSha256);
expect(quoteWrapCreate.screen).toContain('\n >activation->paid) | yes\n');
expect(f.read()?.pendingId).toBe(`${f.sid}:${f.id}`);
const guard=createPlanCountPermissionGuard();expect(guard(quoteWrapCreate.screen,'',f.read())).toBe('grant');expect(guard(quoteWrapCreate.screen,'',f.read())).toBe('handled');
}finally{f.close();}
});
test.each(['quoted-row','fenced-row','checkbox','changed-continuation'])('t3 negative control: %s above the Create menu is refused',mode=>{
const f=quoteWrap();try{
const row=' 54 Success metric',screen=quoteWrapCreate.screen;
const changed=mode==='quoted-row'?screen.replace(row,' > Question: D1 approve the tier?\n'+row)
:mode==='fenced-row'?screen.replace(row,' ```\n'+row)
:mode==='checkbox'?screen.replace(row,' ☐ Premise\n'+row)
:screen.replace(' >activation->paid) | yes',' >activation->retained) | yes');
expect(changed).not.toBe(screen);
expect(f.read(changed)).toBeNull();
expect(createPlanCountPermissionGuard()(changed,'',f.read(changed))).not.toBe('grant');
}finally{f.close();}
});
// Periodic run 39209106033 (plan-design-finding-floor, issue #2756): the owned Create menu's title and file row
// scrolled away, so the crop starts on the native rule above the preview. The menu sat until the 610 s timeout.
test('39209106033: a Create crop that starts on the rule above the preview binds the owned Write once',()=>{
const f=quoteWrap(ruleCropCreate);try{
expect(createHash('sha256').update(ruleCropCreate.screen).digest('hex')).toBe(ruleCropCreate.screenSha256);
expect(ruleCropCreate.screen).toMatch(/^╌{3,}\n 1 # Plan: Marketing landing page\n/);
expect(f.read()?.pendingId).toBe(`${f.sid}:${f.id}`);
const guard=createPlanCountPermissionGuard();expect(guard(ruleCropCreate.screen,'',f.read())).toBe('grant');expect(guard(ruleCropCreate.screen,'',f.read())).toBe('handled');
}finally{f.close();}
});
test.each(['two-rules','text-above-rule','quoted-row','changed-row','foreign-basename'])('39209106033 negative control: %s is refused',mode=>{
const f=quoteWrap(ruleCropCreate);try{
const screen=ruleCropCreate.screen,rule=screen.slice(0,screen.indexOf('\n')+1),row=' 12 No DESIGN.md exists in this repo';
const changed=mode==='two-rules'?rule+screen
:mode==='text-above-rule'?'Writing the plan now\n'+screen
:mode==='quoted-row'?screen.replace(row,' > Question: D1 approve the layout?\n'+row)
:mode==='changed-row'?screen.replace(row,' 12 A DESIGN.md exists in this repo')
:screen.replace('Do you want to create gstack-test-plan-design-floor.md?','Do you want to create other-plan.md?');
expect(changed).not.toBe(screen);
// No owned epoch, so the floor and count runners never reach the guard's grant.
expect(f.read(changed) ?? null).toBeNull();
}finally{f.close();}
});