mirror of
https://github.com/garrytan/gstack.git
synced 2026-10-03 01:46:55 +02:00
v1.91.7.0 feat: add functional QA and pre-publication docs checks (#2983)
* feat: add surface-aware exploratory QA and ship documentation gates * test: preserve delegated QA setup authority after main integration * fix(qa): clarify exploration order and preserve report artifacts * test(qa): follow the shared setup reference directly * refactor(ship): make verification and recovery routes explicit * test(ship): align evidence and review guards with explicit routes * fix(workflows): clarify ship recovery and functional QA evidence * fix(workflows): clarify approval recovery and full QA coverage * refactor(workflows): order review transactions and clarify ship state * fix(ship): clarify final verification and fail closed at publication * fix(evals): attribute native atomic documentation writes * fix(ship): clarify recovery and documentation lifecycle guidance * fix(test): preserve observed native placeholder styling in CI * fix(codex): report watchdog timeouts without a process-exit race * Checkpoint functional QA implementation and workflow validation repairs * Fix documentation and shared-review fixture contracts * docs: clarify judge reuse and evaluation supervision * test: align review evidence and selected case contracts * test: verify append-only documentation checkpoints and recovery * fix: qualify QA workflows and CI validation repairs * fix: launch shared-libs fixture scripts on Windows * fix: qualify QA deadlines, fixture isolation, and shard cleanup * fix: preserve qualified QA and cancellation repairs * fix: enforce functional fixture authority and share strict event decoding * fix: retain free-test evidence and explain recovery * fix: reject malformed native evidence after decoder consolidation * test: use reliable capture for telemetry privacy filters * test: refresh measured quick coverage and document validation costs * Fix native fixture receipts and preserve VM validation evidence * Align negative judge controls with upstream clarity policy * Fix report-only QA preparation and public evidence handling * Clarify QA-only preparation and current-report preservation * Stream Ship quality judgments with an explicit 64k response contract * Validate compact judge reasoning locally with supported wire schema * Align functional QA fixture instructions with evidence acceptance * Bind native browser diagnostics to execution evidence and align review verdicts * Preserve native diagnostic line boundaries * Serialize functional QA evidence from native captures * Keep large QA evidence fixture payload out of Windows argv
This commit is contained in:
Garry Tan authored and GitHub committed
1 parent
65bfb0ce49
commit
dcaea52800
333 files changed
+41755
-7357
No files matched your search
@@ -0,0 +1,198 @@
|
||||
import { afterAll, beforeAll, describe, expect, test } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { computePaidCaseSelection } from '../scripts/test-paid-shards';
|
||||
|
||||
let fixture: string;
|
||||
let observations: any;
|
||||
beforeAll(() => {
|
||||
fixture = fs.mkdtempSync(path.join(os.tmpdir(), 'browse-signals-'));
|
||||
for (const name of ['home', 'tmp', 'state', 'codex']) fs.mkdirSync(path.join(fixture, name));
|
||||
const childScript = path.join(fixture, 'signals.test.ts');
|
||||
fs.writeFileSync(childScript, SCRIPT);
|
||||
const child = spawnSync(process.execPath, ['test', childScript, '--retry', '1'], {
|
||||
cwd: fixture, encoding: 'utf8', timeout: 15_000,
|
||||
env: {
|
||||
PATH: process.env.PATH ?? '', LANG: 'C.UTF-8', NO_COLOR: '1',
|
||||
HOME: path.join(fixture, 'home'), TMPDIR: path.join(fixture, 'tmp'),
|
||||
GSTACK_HOME: path.join(fixture, 'state'), CODEX_HOME: path.join(fixture, 'codex'),
|
||||
EVALS_HERMETIC: '0', SIGNAL_FIXTURE_ROOT: fixture,
|
||||
SIGNAL_REPO_ROOT: path.resolve(import.meta.dir, '..'),
|
||||
},
|
||||
});
|
||||
expect(child.status, child.stderr || child.stdout).toBe(0);
|
||||
observations = JSON.parse(fs.readFileSync(path.join(fixture, 'observations.json'), 'utf8'));
|
||||
}, 20_000);
|
||||
afterAll(() => { if (fixture) fs.rmSync(fixture, { recursive: true, force: true }); });
|
||||
|
||||
test.each(['full', 'pr'] as const)('%s selection binds the captured failure and callback regression to SQL review', profile => {
|
||||
for (const file of ['test/session-runner-browse-errors.test.ts', 'test/fixtures/review-browse-error-ci-36516246523.json']) {
|
||||
expect(computePaidCaseSelection({ profile, env: {}, changedFiles: [file] }).selection)
|
||||
.toEqual({ e2e: ['review-sql-injection'], judges: [] });
|
||||
}
|
||||
});
|
||||
|
||||
describe('native browser-error evidence', () => {
|
||||
test('all JavaScript line terminators bound missing-file diagnostics in execution output and stderr', () => {
|
||||
const rows = observations.sessions.filter(row => row.id.startsWith('line-boundary-'));
|
||||
expect(rows).toHaveLength(8);
|
||||
for (const row of rows) expect(row.result.browseErrors, row.id).toEqual([]);
|
||||
});
|
||||
test.each(['captured-ci', 'separate-lines', 'browser-document', 'browse-document', 'read-document', 'assistant-text', 'transport-metadata'])('%s is not a browser failure', id => {
|
||||
const row = observations.sessions.find(row => row.id === id);
|
||||
expect(row.spawns).toBe(1);
|
||||
expect(row.result.exitReason).toBe('success');
|
||||
expect(row.result.browseErrors).toEqual([]);
|
||||
});
|
||||
|
||||
test.each(['unknown-command', 'snapshot-flag', 'missing-binary', 'start-failure', 'missing-file', 'missing-windows-file'])('%s remains a browser failure in execution output and stderr', id => {
|
||||
for (const suffix of ['', '-stderr']) {
|
||||
const row = observations.sessions.find(row => row.id === id + suffix);
|
||||
expect(row.spawns).toBe(1);
|
||||
expect(row.result.exitReason).toBe('success');
|
||||
expect(row.result.browseErrors).toHaveLength(1);
|
||||
expect(row.result.browseErrors[0]).toContain(row.signal);
|
||||
}
|
||||
});
|
||||
|
||||
test('captured public event reproduces the original serialized-envelope false positive', () => {
|
||||
expect(observations.legacyBrowseError).toBe(observations.capturedBrowseError);
|
||||
expect(observations.legacyBrowseError).toContain('No such file or directory');
|
||||
expect(observations.legacyBrowseError).toContain('tool_use_id');
|
||||
});
|
||||
});
|
||||
|
||||
describe('registered SQL review callback verdict', () => {
|
||||
test('a genuine browser failure triggers Bun’s configured retry and preserves both collector attempts', () => {
|
||||
expect(observations.retry.map(row => row.passed)).toEqual([false, true]);
|
||||
expect(observations.retry.map(row => row.browse_errors)).toEqual([['Server failed to start'], []]);
|
||||
});
|
||||
test.each(['captured-ci', 'browser-error', 'session-error', 'missing-report', 'missing-finding'])('%s agrees with its collector and rejects failures for the native retry', id => {
|
||||
const row = observations.sql.find(row => row.id === id);
|
||||
const passed = id === 'captured-ci';
|
||||
expect(row.registeredName).toBe('review-sql-injection');
|
||||
expect(row.recordings).toHaveLength(1);
|
||||
expect(row.recordings[0].passed).toBe(passed);
|
||||
expect(row.rejected).toBe(!passed);
|
||||
expect(row.recordings[0].browse_errors).toEqual(row.browseErrors);
|
||||
});
|
||||
});
|
||||
|
||||
const SCRIPT = String.raw`import { expect, mock, test } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { EventEmitter } from 'node:events';
|
||||
import { PassThrough } from 'node:stream';
|
||||
const dir = process.env.SIGNAL_FIXTURE_ROOT!;
|
||||
const root = process.env.SIGNAL_REPO_ROOT!;
|
||||
const capture = JSON.parse(fs.readFileSync(path.join(root, 'test/fixtures/review-browse-error-ci-36516246523.json'), 'utf8'));
|
||||
const diagnostics = [
|
||||
['unknown-command', 'Unknown command: unavailable'],
|
||||
['snapshot-flag', 'Unknown snapshot flag: --unavailable'],
|
||||
['missing-binary', 'ERROR: browse binary not found'],
|
||||
['start-failure', 'Server failed to start'],
|
||||
['missing-file', 'No such file or directory: /fixture/browse'],
|
||||
['missing-windows-file', 'No such file or directory: C:\\fixture\\browse.exe'],
|
||||
];
|
||||
function events(output: string, tool = 'Bash') {
|
||||
return [
|
||||
{type:'assistant',message:{content:[{type:'tool_use',id:'fixture-tool',name:tool,input:tool==='Bash'?{command:'browse unavailable'}:{file_path:'/fixture/README.md'}}]}},
|
||||
{type:'user',message:{content:[{type:'tool_result',tool_use_id:'fixture-tool',content:output}]}},
|
||||
];
|
||||
}
|
||||
const cases: any[] = [
|
||||
{id:'captured-ci',events:capture.events},
|
||||
{id:'separate-lines',events:events('No such file or directory: Gemfile\nbrowse')},
|
||||
{id:'browser-document',events:events('No such file or directory: BROWSER.md')},
|
||||
{id:'browse-document',events:events('No such file or directory: browse.md')},
|
||||
{id:'read-document',events:events(diagnostics.map(x=>x[1]).join('\n'),'Read')},
|
||||
{id:'assistant-text',events:[{type:'assistant',message:{content:[{type:'text',text:diagnostics.map(x=>x[1]).join('\n')}]}}]},
|
||||
{id:'transport-metadata',events:[...events('no browser error'),{type:'system',message:diagnostics.map(x=>x[1]).join('\n')}]},
|
||||
...[['lf','\n'],['cr','\r'],['ls','\u2028'],['ps','\u2029']].flatMap(([id,separator])=>[
|
||||
{id:'line-boundary-'+id,events:events('No such file or directory: Gemfile'+separator+'browse')},
|
||||
{id:'line-boundary-'+id+'-stderr',events:events('ok'),stderr:'No such file or directory: Gemfile'+separator+'browse'},
|
||||
]),
|
||||
...diagnostics.flatMap(([id,signal])=>[{id,signal,events:events(signal)},{id:id+'-stderr',signal,events:events('ok'),stderr:signal}]),
|
||||
];
|
||||
let active: any;
|
||||
mock.module('child_process',()=>({
|
||||
spawn(command: string,args: string[],options: any) {
|
||||
if(command!=='claude')throw Error('Unexpected executable');
|
||||
const x=active;x.spawns++;
|
||||
queueMicrotask(()=>{
|
||||
for(const event of x.scenario.events)x.child.stdout.write(JSON.stringify(event)+'\n');
|
||||
x.child.stdout.end(JSON.stringify({type:'result',subtype:'success',is_error:false,result:'fixture result',num_turns:1,total_cost_usd:0})+'\n');
|
||||
x.child.stderr.end(x.scenario.stderr??'');
|
||||
x.child.exitCode=0;x.child.emit('exit',0,null);
|
||||
});
|
||||
return x.child;
|
||||
},
|
||||
spawnSync(){return {status:1,stdout:Buffer.alloc(0),stderr:Buffer.alloc(0)};},
|
||||
execFileSync(){throw Error('Unexpected synchronous executable');},
|
||||
}));
|
||||
mock.module(path.join(root,'scripts/test-strict-output.ts'),()=>({killProcessGroup(child: any){if(child!==active.child)throw Error('Unknown child');}}));
|
||||
const {runSkillTest}=await import(path.join(root,'test/helpers/session-runner.ts'));
|
||||
const {recordE2E}=await import(path.join(root,'test/helpers/e2e-helpers.ts'));
|
||||
test('exercise the actual runner and registered SQL callback without a provider',async()=>{
|
||||
const sessions: any[]=[];
|
||||
for(const scenario of cases) {
|
||||
const child=Object.assign(new EventEmitter(),{pid:undefined,stdin:new PassThrough(),stdout:new PassThrough(),stderr:new PassThrough(),exitCode:null});
|
||||
const x=active={scenario,child,spawns:0};
|
||||
const cwd=path.join(dir,scenario.id);fs.mkdirSync(cwd);
|
||||
const result=await runSkillTest({prompt:'local native-event replay',workingDirectory:cwd,timeout:1000,testName:scenario.id,model:'fixture-no-provider'});
|
||||
sessions.push({id:scenario.id,signal:scenario.signal,spawns:x.spawns,result});
|
||||
}
|
||||
const source=fs.readFileSync(path.join(root,'test/skill-e2e-review.test.ts'),'utf8');
|
||||
const start=source.indexOf("testConcurrentIfSelected('review-sql-injection',");
|
||||
const ending='}, CAPTURE_MS + REVIEW_FINALIZE_MS);';
|
||||
const end=source.indexOf(ending,start);
|
||||
expect(start).toBeGreaterThanOrEqual(0);expect(end).toBeGreaterThan(start);
|
||||
const registration=source.slice(start,end+ending.length);
|
||||
const sql: any[]=[];
|
||||
for(const id of ['captured-ci','browser-error','session-error','missing-report','missing-finding']) {
|
||||
const reviewDir=path.join(dir,'sql-'+id);fs.mkdirSync(reviewDir);
|
||||
if(id!=='missing-report')fs.writeFileSync(path.join(reviewDir,'review-output.md'),id==='missing-finding'?'Nothing found':'SQL injection in user input');
|
||||
const result=structuredClone(sessions.find(row=>row.id==='captured-ci').result);
|
||||
if(id!=='captured-ci')result.browseErrors=[];
|
||||
if(id==='browser-error')result.browseErrors=['Server failed to start'];
|
||||
if(id==='session-error')result.exitReason='error_api';
|
||||
const recordings: any[]=[];let callback: any;let registeredName: string='';
|
||||
const register=(name: string,fn: any)=>{registeredName=name;callback=fn;};
|
||||
const transpiled=new Bun.Transpiler({loader:'ts'}).transformSync(registration);
|
||||
new Function('testConcurrentIfSelected','runSkillTest','reviewDir','CAPTURE_MS','REVIEW_FINALIZE_MS','runId','logCost','recordE2E','evalCollector','expect','fs','path',transpiled)(register,async()=>result,reviewDir,1234,5000,'fixture-run',()=>{},recordE2E,{addTest(row: any){recordings.push(row);}},expect,fs,path);
|
||||
expect(typeof callback).toBe('function');
|
||||
let rejected=false;try{await callback();}catch{rejected=true;}
|
||||
sql.push({id,registeredName,recordings,rejected,browseErrors:result.browseErrors});
|
||||
}
|
||||
const legacyBrowseError=capture.events.map(event=>JSON.stringify(event)).join('\n').match(/no such file or directory.*browse/i)?.[0].slice(0,200);
|
||||
fs.writeFileSync(path.join(dir,'observations.json'),JSON.stringify({sessions,sql,legacyBrowseError,capturedBrowseError:capture.source.browseErrors[0]},null,2)+'\n',{mode:0o600});
|
||||
},10000);
|
||||
const retryDir=path.join(dir,'native-sql-retry');fs.mkdirSync(retryDir);
|
||||
fs.writeFileSync(path.join(retryDir,'review-output.md'),'SQL injection in user input');
|
||||
const retrySource=fs.readFileSync(path.join(root,'test/skill-e2e-review.test.ts'),'utf8');
|
||||
const retryStart=retrySource.indexOf("testConcurrentIfSelected('review-sql-injection',");
|
||||
const retryEnding='}, CAPTURE_MS + REVIEW_FINALIZE_MS);';
|
||||
const retryEnd=retrySource.indexOf(retryEnding,retryStart);
|
||||
expect(retryStart).toBeGreaterThanOrEqual(0);expect(retryEnd).toBeGreaterThan(retryStart);
|
||||
let retryAttempts=0;
|
||||
const retryRecordings: any[]=[];
|
||||
const registerRetry=(name: string,fn: any)=>test(name,async()=>{
|
||||
try{await fn();}finally{
|
||||
const file=path.join(dir,'observations.json');
|
||||
const data=JSON.parse(fs.readFileSync(file,'utf8'));
|
||||
data.retry=retryRecordings;
|
||||
fs.writeFileSync(file,JSON.stringify(data,null,2)+'\n',{mode:0o600});
|
||||
}
|
||||
});
|
||||
const retryResult=async()=>{
|
||||
retryAttempts++;
|
||||
const data=JSON.parse(fs.readFileSync(path.join(dir,'observations.json'),'utf8'));
|
||||
const result=data.sessions.find(row=>row.id==='captured-ci').result;
|
||||
result.browseErrors=retryAttempts===1?['Server failed to start']:[];
|
||||
return result;
|
||||
};
|
||||
const retryJs=new Bun.Transpiler({loader:'ts'}).transformSync(retrySource.slice(retryStart,retryEnd+retryEnding.length));
|
||||
new Function('testConcurrentIfSelected','runSkillTest','reviewDir','CAPTURE_MS','REVIEW_FINALIZE_MS','runId','logCost','recordE2E','evalCollector','expect','fs','path',retryJs)(registerRetry,retryResult,retryDir,1234,5000,'fixture-retry',()=>{},recordE2E,{addTest(row: any){retryRecordings.push(row);}},expect,fs,path);
|
||||
`;
|
||||
Reference in new issue
Block a user