v1.91.7.0 feat: add functional QA and pre-publication docs checks (#2983)

* feat: add surface-aware exploratory QA and ship documentation gates

* test: preserve delegated QA setup authority after main integration

* fix(qa): clarify exploration order and preserve report artifacts

* test(qa): follow the shared setup reference directly

* refactor(ship): make verification and recovery routes explicit

* test(ship): align evidence and review guards with explicit routes

* fix(workflows): clarify ship recovery and functional QA evidence

* fix(workflows): clarify approval recovery and full QA coverage

* refactor(workflows): order review transactions and clarify ship state

* fix(ship): clarify final verification and fail closed at publication

* fix(evals): attribute native atomic documentation writes

* fix(ship): clarify recovery and documentation lifecycle guidance

* fix(test): preserve observed native placeholder styling in CI

* fix(codex): report watchdog timeouts without a process-exit race

* Checkpoint functional QA implementation and workflow validation repairs

* Fix documentation and shared-review fixture contracts

* docs: clarify judge reuse and evaluation supervision

* test: align review evidence and selected case contracts

* test: verify append-only documentation checkpoints and recovery

* fix: qualify QA workflows and CI validation repairs

* fix: launch shared-libs fixture scripts on Windows

* fix: qualify QA deadlines, fixture isolation, and shard cleanup

* fix: preserve qualified QA and cancellation repairs

* fix: enforce functional fixture authority and share strict event decoding

* fix: retain free-test evidence and explain recovery

* fix: reject malformed native evidence after decoder consolidation

* test: use reliable capture for telemetry privacy filters

* test: refresh measured quick coverage and document validation costs

* Fix native fixture receipts and preserve VM validation evidence

* Align negative judge controls with upstream clarity policy

* Fix report-only QA preparation and public evidence handling

* Clarify QA-only preparation and current-report preservation

* Stream Ship quality judgments with an explicit 64k response contract

* Validate compact judge reasoning locally with supported wire schema

* Align functional QA fixture instructions with evidence acceptance

* Bind native browser diagnostics to execution evidence and align review verdicts

* Preserve native diagnostic line boundaries

* Serialize functional QA evidence from native captures

* Keep large QA evidence fixture payload out of Windows argv
This commit is contained in:
Garry Tan authored and GitHub committed 2026-09-29 06:07:35 -07:00
1 parent 65bfb0ce49
commit dcaea52800
333 files changed
+41755 -7357

No files matched your search

+14 -11
View File
@@ -39,7 +39,7 @@ process.stdin.setRawMode(true);
process.stdin.resume();
process.stdin.on('data', bytes => log('input', {data:bytes.toString()}));
process.on('SIGINT', () => log('sigint')); // exercise the owned forced-exit fallback
process.stdout.write('Counting lifecycle fixture is ready.\n');
process.stdout.write('COUNT_TIMEOUT_FIXTURE_READY\n');
setInterval(() => {}, 1000);
`, { mode: 0o755 });
fs.writeFileSync(worker, `import {test} from 'bun:test';\nimport * as fs from 'node:fs';\nimport {runPlanSkillCounting} from ${JSON.stringify(helper)};\n` + String.raw`
@@ -52,12 +52,13 @@ test('owned counting timeout', async () => {
try {
const observation = await runPlanSkillCounting({skillName:'plan-design-review', slashCommand:'/plan-design-review',
followUpPrompt:'# Timeout lifecycle fixture\nReview this plan.', isLastStep0AUQ:()=>false,
reviewCountCeiling:8, timeoutMs:18000, env:{TIMEOUT_INVOCATION:String(invocation),TIMEOUT_EVENTS:process.env.TIMEOUT_EVENTS}});
reviewCountCeiling:8, timeoutMs:8000, startupReadyMarker:'COUNT_TIMEOUT_FIXTURE_READY',
env:{TIMEOUT_INVOCATION:String(invocation),TIMEOUT_EVENTS:process.env.TIMEOUT_EVENTS}});
const ready = fs.readFileSync(process.env.TIMEOUT_EVENTS,'utf8').trim().split('\n').map(line=>JSON.parse(line)).find(e=>e.event==='ready'&&e.invocation===invocation);
log('returned', invocation, {elapsed:Date.now()-start,outcome:observation.outcome,fixtureGone:!fs.existsSync(ready.cwd)});
throw new Error('HELPER_TIMEOUT_'+invocation);
} finally { log('finally', invocation); }
}, 18000);
}, 8000);
`);
let rows: Event[] = [];
let child: ReturnType<typeof Bun.spawn> | undefined;
@@ -89,7 +90,7 @@ test('owned counting timeout', async () => {
expect(finished[0]!.at).toBeLessThanOrEqual(starts[1]!.at);
for (const event of returned) {
expect(event.outcome).toBe('timeout');
expect(event.elapsed).toBeLessThan(18000);
expect(event.elapsed).toBeLessThan(8000);
expect(event.fixtureGone).toBe(true);
}
for (const event of ready) {
@@ -97,7 +98,7 @@ test('owned counting timeout', async () => {
const body = starts.find(e => e.invocation === event.invocation)!;
const inputs = rows.filter(e => e.event === 'input' && e.invocation === event.invocation);
expect(inputs.map(e => e.data).join('')).toBe('/plan-design-review\r');
expect(inputs.every(e => e.at - body.at < 13000)).toBe(true);
expect(inputs.every(e => e.at - body.at < 3000)).toBe(true);
}
} finally {
if (watchdog) clearTimeout(watchdog);
@@ -138,6 +139,7 @@ process.stdin.on('data',data=>{
process.stdout.write('\x1b[2J\x1b[HWhich remedy should be used?\r\n❯1.First remedy\r\n2.Second remedy\r\n');
});
process.on('SIGINT',()=>{log('sigint');process.exit(0)});
process.stdout.write('COUNT_BOUNDARY_FIXTURE_READY\n');
setInterval(()=>{},1000);
`, { mode: 0o755 });
fs.writeFileSync(worker, `import {mock} from 'bun:test';\nimport * as fs from 'node:fs';\n` +
@@ -148,7 +150,7 @@ const log=(event,extra={})=>fs.appendFileSync(process.env.BOUNDARY_EVENTS,JSON.s
if(mode==='boot') {
const sleep=Bun.sleep.bind(Bun);
Bun.sleep=async ms=>{
if(typeof ms==='number' && ms>1000 && ms<8000) {
if(typeof ms==='number' && ms>250 && ms<1000) {
log('early-clipped-wake',{requested:ms});
return sleep(Math.max(0,ms-250));
}
@@ -157,17 +159,18 @@ if(mode==='boot') {
}
if(mode==='screen') mock.module(screenModule,()=>({createPtyScreen:async(...args)=>{
const screen=await originalScreen(...args);
return {...screen,read:async()=>{reads++; await Bun.sleep(Math.max(0,start+13200-Date.now()));return screen.read();}};
return {...screen,read:async()=>{reads++; await Bun.sleep(Math.max(0,start+3200-Date.now()));return screen.read();}};
}}));
` + `const {runPlanSkillCounting}=await import(${JSON.stringify(helper)});\n` + String.raw`
start=Date.now();log('body');
const observation=await runPlanSkillCounting({skillName:'plan-design-review',slashCommand:'/plan-design-review',
followUpPrompt:'Review the deadline fixture.',isLastStep0AUQ:()=>false,reviewCountCeiling:8,timeoutMs:mode==='boot'?12000:18000,
followUpPrompt:'Review the deadline fixture.',isLastStep0AUQ:()=>false,reviewCountCeiling:8,timeoutMs:mode==='boot'?6000:8000,
...(mode==='boot'?{}:{startupReadyMarker:'COUNT_BOUNDARY_FIXTURE_READY'}),
pickAUQ:(_routing,_active,context)=>{
const ready=fs.readFileSync(process.env.BOUNDARY_EVENTS,'utf8').trim().split('\n').map(line=>JSON.parse(line)).find(event=>event.event==='ready');
if(!Object.isFrozen(context)||context.cwd!==ready.cwd||!Number.isFinite(context.deadlineAt)||context.deadlineAt<=Date.now()||context.deadlineAt>start+18000)
if(!Object.isFrozen(context)||context.cwd!==ready.cwd||!Number.isFinite(context.deadlineAt)||context.deadlineAt<=Date.now()||context.deadlineAt>start+8000)
throw new Error('Picker did not receive its owned fixture and bounded deadline');
log('picker');while(Date.now()-start<12800){};return 2;
log('picker');while(Date.now()-start<2800){};return 2;
},
env:{BOUNDARY_EVENTS:process.env.BOUNDARY_EVENTS}});
const events=fs.readFileSync(process.env.BOUNDARY_EVENTS,'utf8').trim().split('\n').map(line=>JSON.parse(line));
@@ -188,7 +191,7 @@ log('returned',{outcome:observation.outcome,elapsed:Date.now()-start,reads,fixtu
const events=fs.readFileSync(eventPath,'utf8').trim().split('\n').map(line=>JSON.parse(line));
const returned=events.find(e=>e.event==='returned');
expect(returned.outcome).toBe('timeout'); expect(returned.fixtureGone).toBe(true);
expect(returned.elapsed).toBeLessThan(mode==='boot'?12000:18000);
expect(returned.elapsed).toBeLessThan(mode==='boot'?6000:8000);
const input=events.filter(e=>e.event==='input').map(e=>e.data).join('');
expect(input).toBe(mode==='boot'?'':mode==='screen'?'/plan-design-review\r':'/plan-design-review\r2');
if(mode==='boot') expect(events.some(e=>e.event==='early-clipped-wake')).toBe(true);