Files
gstack/test/autoplan-pending-artifact.test.ts
T
Garry Tan dcaea52800 v1.91.7.0 feat: add functional QA and pre-publication docs checks (#2983)
* feat: add surface-aware exploratory QA and ship documentation gates

* test: preserve delegated QA setup authority after main integration

* fix(qa): clarify exploration order and preserve report artifacts

* test(qa): follow the shared setup reference directly

* refactor(ship): make verification and recovery routes explicit

* test(ship): align evidence and review guards with explicit routes

* fix(workflows): clarify ship recovery and functional QA evidence

* fix(workflows): clarify approval recovery and full QA coverage

* refactor(workflows): order review transactions and clarify ship state

* fix(ship): clarify final verification and fail closed at publication

* fix(evals): attribute native atomic documentation writes

* fix(ship): clarify recovery and documentation lifecycle guidance

* fix(test): preserve observed native placeholder styling in CI

* fix(codex): report watchdog timeouts without a process-exit race

* Checkpoint functional QA implementation and workflow validation repairs

* Fix documentation and shared-review fixture contracts

* docs: clarify judge reuse and evaluation supervision

* test: align review evidence and selected case contracts

* test: verify append-only documentation checkpoints and recovery

* fix: qualify QA workflows and CI validation repairs

* fix: launch shared-libs fixture scripts on Windows

* fix: qualify QA deadlines, fixture isolation, and shard cleanup

* fix: preserve qualified QA and cancellation repairs

* fix: enforce functional fixture authority and share strict event decoding

* fix: retain free-test evidence and explain recovery

* fix: reject malformed native evidence after decoder consolidation

* test: use reliable capture for telemetry privacy filters

* test: refresh measured quick coverage and document validation costs

* Fix native fixture receipts and preserve VM validation evidence

* Align negative judge controls with upstream clarity policy

* Fix report-only QA preparation and public evidence handling

* Clarify QA-only preparation and current-report preservation

* Stream Ship quality judgments with an explicit 64k response contract

* Validate compact judge reasoning locally with supported wire schema

* Align functional QA fixture instructions with evidence acceptance

* Bind native browser diagnostics to execution evidence and align review verdicts

* Preserve native diagnostic line boundaries

* Serialize functional QA evidence from native captures

* Keep large QA evidence fixture payload out of Windows argv
2026-09-29 06:07:35 -07:00

208 lines
15 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import { afterEach, expect, test } from 'bun:test';
import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import { pathToFileURL } from 'node:url';
import fixture from './fixtures/autoplan-pending-artifact-ae.json';
import { autoplanArtifactPermissionInput, pendingAutoplanArtifactPermissionInput, autoplanArtifactMenuKey } from './helpers/autoplan-artifact-permission';
import { createAutoplanArtifactRecorder, recordAutoplanArtifact, readPendingAutoplanArtifact } from './helpers/autoplan-artifact-recorder';
import type { NativePublicToolEvent } from './helpers/plan-count-transcript';
const roots:string[]=[];
afterEach(()=>{for(const root of roots.splice(0))fs.rmSync(root,{recursive:true,force:true});});
function replay(relative='ceo-plans/2026-09-09-user-dashboard.md',clock=Date.now) {
const root=fs.mkdtempSync(path.join(os.tmpdir(),'gstack-pending-artifact-test-'));roots.push(root);
const cwd=path.join(root,path.basename(fixture.cwd)),ownedStateRoot=path.join(root,'home','.gstack'),config=path.join(root,'config');
fs.mkdirSync(cwd);const file=path.join(ownedStateRoot,'projects',path.basename(cwd),relative);
fs.mkdirSync(path.dirname(file),{recursive:true});fs.writeFileSync(file,fixture.before);
const native=path.join(config,'projects','fixture',fixture.sessionId+'.jsonl');fs.mkdirSync(path.dirname(native),{recursive:true});fs.writeFileSync(native,'');
const publicTools=structuredClone(fixture.events) as NativePublicToolEvent[];
for(const e of publicTools)if(e.input)e.input.file_path=file;
const recorder=createAutoplanArtifactRecorder(cwd,config,ownedStateRoot);
// Synthetic hook: only its identity/path are retained. Neither this input
// nor the displayed additions are claimed to reproduce the unpublished body.
const event={hook_event_name:'PreToolUse',tool_name:'Edit',session_id:fixture.sessionId,tool_use_id:'synthetic-current-edit',
cwd,transcript_path:native,tool_input:{file_path:file,old_string:'Synthetic old content',new_string:'Synthetic new content',replace_all:false}};
const record=(change:Record<string,unknown>={})=>recordAutoplanArtifact(JSON.stringify({...event,...change}),recorder.file,cwd,config,ownedStateRoot);
record();
const observedAt=clock();
const context={cwd,ownedStateRoot,commandStartedAt:fixture.commandStartedAt,now:observedAt,viewportCapturedAt:observedAt,
transcriptStatus:'ready',publicTools,pending:readPendingAutoplanArtifact(recorder.file,cwd,config,ownedStateRoot,fixture.commandStartedAt,publicTools)};
const screen=fixture.viewport.replaceAll(path.basename(fixture.file),path.basename(file));
roots.push(path.dirname(recorder.file));
return {root,file,native,config,recorder,event,record,context,screen};
}
const pick=(r:ReturnType<typeof replay>,seen=new Set<string>())=>pendingAutoplanArtifactPermissionInput(r.screen,r.context,seen);
test('actual public pane stays blocked without hook identity; synthetic owned metadata enables only one option',()=>{
const r=replay();
expect(autoplanArtifactPermissionInput(r.screen,r.context,new Set())).toBeNull();
expect(pick({...r,context:{...r.context,pending:undefined}})).toBeNull();
expect(pick(r)).toEqual({input:'1\r',signature:fixture.sessionId+':synthetic-current-edit',file:r.file});
expect(pick(r,new Set([pick(r)!.signature]))).toBeNull();
expect(JSON.stringify(r.context.pending)).not.toContain('Synthetic old content');
expect(r.context.publicTools).toHaveLength(fixture.events.length);
});
test('all130 actual published tool events preserve the same metadata-only fallback boundary',()=>{
const r=replay();r.context.publicTools=structuredClone(fixture.allPublicTools) as NativePublicToolEvent[];
for(const e of r.context.publicTools)if(e.input?.file_path===fixture.file)e.input.file_path=r.file;
expect(r.context.publicTools).toHaveLength(130);
expect(r.context.publicTools.filter(e=>e.kind==='use' && ['Write','Edit'].includes(e.name??''))).toHaveLength(39);
expect(pick(r)?.input).toBe('1\r');
});
test('one clock sample keeps all130-event replay coherent across a millisecond boundary without allowing future viewports',()=>{
let first:number|undefined,reads=0;
const clock=()=>(first??=Date.now())+reads++;
const r=replay(undefined,clock);
r.context.publicTools=structuredClone(fixture.allPublicTools) as NativePublicToolEvent[];
for(const event of r.context.publicTools)if(event.input?.file_path===fixture.file)event.input.file_path=r.file;
expect(reads).toBe(1);
expect(r.context.viewportCapturedAt).toBe(r.context.now);
expect(r.context.publicTools).toHaveLength(130);
expect(Math.floor(fs.statSync(r.file).mtimeMs)).toBeLessThanOrEqual(Date.parse(r.context.pending!.timestamp));
expect(pick(r)?.input).toBe('1\r');
const future={...r.context,viewportCapturedAt:clock()};
expect(future.viewportCapturedAt).toBe(r.context.now+1);
expect(pick({...r,context:future})).toBeNull();
expect(pick({...r,context:{...future,now:future.viewportCapturedAt}})?.input).toBe('1\r');
expect(pick({...r,context:{...r.context,viewportCapturedAt:Date.parse(r.context.pending!.timestamp)-1}})).toBeNull();
});
test('completed or published requests and newer identities on an old granted viewport remain closed',()=>{
const r=replay(),first=pick(r)!;
const seen=new Set([first.signature,autoplanArtifactMenuKey(r.screen)]);
r.record({hook_event_name:'PostToolUse'});
expect(readPendingAutoplanArtifact(r.recorder.file,r.context.cwd,r.config,r.context.ownedStateRoot,r.context.commandStartedAt,r.context.publicTools)).toBeUndefined();
r.record({tool_use_id:'newer-request'});
r.context.now=Date.now();r.context.viewportCapturedAt=r.context.now;
r.context.pending=readPendingAutoplanArtifact(r.recorder.file,r.context.cwd,r.config,r.context.ownedStateRoot,r.context.commandStartedAt,r.context.publicTools);
expect(pick(r,seen)).toBeNull();
r.context.publicTools.push({kind:'result',sessionId:fixture.sessionId,toolUseId:'newer-request',timestamp:new Date().toISOString(),isError:false});
expect(pick(r)).toBeNull();
});
test('hook after viewport, invalid clocks, future/stale/foreign IDs and missing success cannot authorize input',()=>{
const changes:Array<(r:ReturnType<typeof replay>)=>void>=[
r=>{r.context.viewportCapturedAt=Date.parse(r.context.pending!.timestamp)-1;},
r=>{r.context.now=NaN;},r=>{r.context.now=Infinity;},r=>{r.context.viewportCapturedAt=NaN;},
r=>{r.context.viewportCapturedAt=r.context.now+1;},
r=>{r.context.pending!.timestamp=new Date(r.context.now+10000).toISOString();},
r=>{r.context.pending!.timestamp=new Date(r.context.commandStartedAt-1).toISOString();},
r=>{r.context.pending!.sessionId='foreign';},r=>{r.context.pending!.toolUseId='';},
r=>{r.context.pending!.toolUseId='invalid:id';},r=>{r.context.pending!.file=42 as any;},r=>{r.context.publicTools=[];},
r=>{r.context.transcriptStatus='error';},
r=>{for(const e of r.context.publicTools)if(e.kind==='result')e.isError=true;},
r=>{r.context.publicTools.push({...r.context.publicTools[0]!,toolUseId:'unresolved-concurrent',timestamp:new Date().toISOString()});},
r=>{r.context.publicTools.push({...r.context.publicTools[0]!,toolUseId:r.context.pending!.toolUseId,timestamp:new Date().toISOString()});},
r=>{r.context.publicTools.push({...r.context.publicTools.at(-1)!,sessionId:'sibling'});},
];
for(const change of changes){const r=replay();change(r);expect(pick(r),change.toString()).toBeNull();}
});
test('changed, foreign and symlink files are rejected; all existing owned artifact layouts stay scoped',()=>{
for(const relative of ['ceo-plans/2026-09-09-user-dashboard.md','main-test-plan-20260909-220000.md','main-eng-review-test-plan-20260909-220000.md'])expect(pick(replay(relative))?.input).toBe('1\r');
for(const relative of ['other.md','config.yaml','tasks.jsonl','../sibling/ceo-plans/2026-09-09-user-dashboard.md'])expect(pick(replay(relative))).toBeNull();
let r=replay();fs.writeFileSync(r.file,'Changed unrelated content');expect(pick(r)).toBeNull();
r=replay();fs.utimesSync(r.file,new Date(r.context.now+10000),new Date(r.context.now+10000));expect(pick(r)).toBeNull();
if(process.platform!=='win32'){
r=replay();const sibling=r.file+'.sibling';fs.renameSync(r.file,sibling);fs.symlinkSync(sibling,r.file);expect(pick(r)).toBeNull();
}
r=replay();r.context.ownedStateRoot=path.join(r.root,'ambient-home');expect(pick(r)).toBeNull();
});
test('only a complete current native menu and current-file deleted/context rows support pending metadata',()=>{
const changes=[
(s:string)=>'Example:\n'+s,(s:string)=>'```\n'+s+'```',
(s:string)=>s.split('\n').map(l=>'> '+l).join('\n'),
(s:string)=>s.replace(' ❯ 1. Yes',' ❯ 1. Yes, always allow'),
(s:string)=>s.replace(' ❯ 1. Yes',' 1. Yes').replace(' 2. Yes',' ❯ 2. Yes'),
(s:string)=>s.replace(' 3. No',' 3. No\n 4. Run a command'),
(s:string)=>s.replace('2026-09-09-user-dashboard.md?','foreign.md?'),
(s:string)=>s.replace('Esc to cancel · Tab to amend','Enter to select'),
(s:string)=>s+'\nPlease run the extra work.',
(s:string)=>s.replace(' -than the latest',' -unrelated cropped text'),
(s:string)=>s.replace(' 50 -- **Retry.**',' 50 -- **Unrelated deletion.**'),
(s:string)=>s.slice(s.indexOf(' Do you want')),
];
for(const change of changes){const r=replay();r.screen=change(r.screen);expect(pick(r),change.toString()).toBeNull();}
});
test('queued unrelated public tools do not confer permission or block the current owned edit',()=>{
const r=replay();r.context.publicTools.push({kind:'use',sessionId:fixture.sessionId,toolUseId:'queued-bash',name:'Bash',
timestamp:new Date(r.context.now).toISOString(),input:{command:'echo queued'}});
expect(pick(r)?.input).toBe('1\r');
r.context.publicTools.at(-1)!.name='Write';expect(pick(r)).toBeNull();
});
for (const [line, numbered, next, continuation] of [
[7, ' 7 ', ' 8 ', ' '], [17, ' 17 ', ' 18 ', ' '],
[116, ' 116 ', ' 117 ', ' '], [1024, ' 1024 ', ' 1025 ', ' '],
] as const) test(`legacy pending deletion line ${line} binds leading and wrapped fragments to its numbered column`, () => {
const r = replay();
expect(r.context.pending?.editDigest).toBeUndefined();
const before = Array.from({ length: line - 2 }, (_, n) => `Context ${n}`)
.concat('Head before crop tail', 'Old complete row', 'Context').join('\n');
fs.writeFileSync(r.file, before);
const at = new Date(Date.parse(r.context.pending!.timestamp) - 1); fs.utimesSync(r.file, at, at);
const menu = r.screen.slice(r.screen.indexOf(' Do you want'));
const rows = `${continuation}-tail\n${numbered}-Old complete\n${continuation}- row\n` +
`${numbered}+New complete\n${continuation}+ row\n${next} Context\n`;
const pane = rows + '╌'.repeat(20) + '\n' + menu;
r.screen = pane;
expect(pick(r)?.input).toBe('1\r');
expect(pick(r, new Set([pick(r)!.signature]))).toBeNull();
for (const invalid of [
pane.replaceAll(continuation + '-', continuation.slice(1) + '-'),
pane.replaceAll(continuation + '-', ' ' + continuation + '-'),
pane.replace(continuation + '- row', continuation + '+ row'),
pane.replace(next + ' Context', ' ' + next + ' Context'),
pane.replace('Old complete', 'Unrelated deleted'),
pane.replace(continuation + '-tail', continuation + '-foreign suffix'),
pane.replaceAll(numbered, ' 0 '),
]) { r.screen = invalid; expect(pick(r), invalid).toBeNull(); }
});
test.skipIf(process.platform==='win32')('real launcher installs only opt-in owned hooks and removes records on close or early exit',async()=>{
const root=fs.mkdtempSync(path.join(os.tmpdir(),'gstack-artifact-launch-'));roots.push(root);
const fake=path.join(root,'fake-claude');fs.writeFileSync(fake,`#!${process.execPath}\n`+String.raw`
import * as fs from 'node:fs';
fs.writeFileSync(process.env.ARTIFACT_RECORD,JSON.stringify({pid:process.pid,args:process.argv.slice(2)}));
if(process.env.ARTIFACT_FAIL==='1')process.exit(19);
process.stdout.write('ARTIFACT_READY\n');process.stdin.resume();
`,{mode:0o755});
const runner=pathToFileURL(path.join(import.meta.dir,'helpers/claude-pty-runner.ts')).href;
for(const variant of ['enabled','approval','disabled','explicit-home','explicit-config','early-exit']){
const cwd=path.join(root,variant);fs.mkdirSync(cwd);const result=path.join(cwd,'result.json');
const extra=variant==='explicit-home'?{HOME:cwd}:variant==='explicit-config'?{CLAUDE_CONFIG_DIR:cwd}:{};
const worker=path.join(cwd,'worker.ts');fs.writeFileSync(worker,`
import * as fs from 'node:fs';
import {launchClaudePty,resolveClaudeBinary} from ${JSON.stringify(runner)};
if(resolveClaudeBinary()!==${JSON.stringify(fake)})throw Error('fake binding');
const session=await launchClaudePty({cwd:${JSON.stringify(cwd)},seedSkills:true,observeAutoplanArtifacts:${variant!=='disabled'},approveAutoplanArtifactEdits:${variant==='approval'||variant.startsWith('explicit-')},timeoutMs:8000,
env:${JSON.stringify({...extra,ARTIFACT_RECORD:result,ARTIFACT_FAIL:variant==='early-exit'?'1':'0'})}});
try {try{await session.waitFor('ARTIFACT_READY',{timeoutMs:2000,pollMs:20});}catch(e){if(${variant!=='early-exit'})throw e;}
const r=JSON.parse(fs.readFileSync(${JSON.stringify(result)},'utf8'));
r.file=session.pendingAutoplanArtifactFile??null;r.stateRoot=session.hermeticSkillStateRoot??null;
r.canStart=typeof session.startAutoplanArtifactEditApproval==='function';
if(${variant==='approval'}){const start=Date.now();session.startAutoplanArtifactEditApproval(start);r.started=JSON.parse(fs.readFileSync(r.file,'utf8')).approvalStartedAt===start;}
r.exists=r.file?fs.existsSync(r.file):false;fs.writeFileSync(${JSON.stringify(result)},JSON.stringify(r));
} finally {await session.close();}
`);
const child=Bun.spawn([process.execPath,worker],{env:{...process.env,BROWSE_TERMINAL_BINARY:fake,EVALS_HERMETIC:'1'},stdout:'pipe',stderr:'pipe'});
const timer=setTimeout(()=>child.kill('SIGKILL'),15000);
try{const [code,out,err]=await Promise.all([child.exited,new Response(child.stdout).text(),new Response(child.stderr).text()]);expect(code,out+err).toBe(0);}finally{clearTimeout(timer);}
const resultData=JSON.parse(fs.readFileSync(result,'utf8')),enabled=['enabled','approval','early-exit'].includes(variant);
expect(resultData.canStart).toBe(variant==='approval');
if(variant==='approval')expect(resultData.started).toBe(true);
expect(Boolean(resultData.file)).toBe(enabled);expect(resultData.exists).toBe(enabled);
if(enabled){const settings=JSON.parse(resultData.args[resultData.args.indexOf('--settings')+1]);
expect(Object.keys(settings.hooks).sort()).toEqual(['PostToolUse','PostToolUseFailure','PreToolUse']);
for(const entries of Object.values(settings.hooks) as any[]){expect(entries).toHaveLength(1);expect(entries[0].matcher).toBe('^(Write|Edit)$');expect(entries[0].hooks[0].timeout).toBe(5);expect(entries[0].hooks[0].command).toContain(resultData.stateRoot);expect(entries[0].hooks[0].command.includes('--approve-edits')).toBe(variant==='approval');}
expect(fs.existsSync(resultData.file)).toBe(false);
}else expect(resultData.args).not.toContain('--settings');
expect(()=>process.kill(resultData.pid,0)).toThrow();
}
},90000);