CalvinBackup/gstack
1
0 Fork 0
mirror of https://github.com/garrytan/gstack.git synced 2026-10-03 18:06:54 +02:00
Code Issues Packages Projects Releases Wiki Activity

v1.91.7.0 feat: add functional QA and pre-publication docs checks (#2983)

Browse Source 
* feat: add surface-aware exploratory QA and ship documentation gates

* test: preserve delegated QA setup authority after main integration

* fix(qa): clarify exploration order and preserve report artifacts

* test(qa): follow the shared setup reference directly

* refactor(ship): make verification and recovery routes explicit

* test(ship): align evidence and review guards with explicit routes

* fix(workflows): clarify ship recovery and functional QA evidence

* fix(workflows): clarify approval recovery and full QA coverage

* refactor(workflows): order review transactions and clarify ship state

* fix(ship): clarify final verification and fail closed at publication

* fix(evals): attribute native atomic documentation writes

* fix(ship): clarify recovery and documentation lifecycle guidance

* fix(test): preserve observed native placeholder styling in CI

* fix(codex): report watchdog timeouts without a process-exit race

* Checkpoint functional QA implementation and workflow validation repairs

* Fix documentation and shared-review fixture contracts

* docs: clarify judge reuse and evaluation supervision

* test: align review evidence and selected case contracts

* test: verify append-only documentation checkpoints and recovery

* fix: qualify QA workflows and CI validation repairs

* fix: launch shared-libs fixture scripts on Windows

* fix: qualify QA deadlines, fixture isolation, and shard cleanup

* fix: preserve qualified QA and cancellation repairs

* fix: enforce functional fixture authority and share strict event decoding

* fix: retain free-test evidence and explain recovery

* fix: reject malformed native evidence after decoder consolidation

* test: use reliable capture for telemetry privacy filters

* test: refresh measured quick coverage and document validation costs

* Fix native fixture receipts and preserve VM validation evidence

* Align negative judge controls with upstream clarity policy

* Fix report-only QA preparation and public evidence handling

* Clarify QA-only preparation and current-report preservation

* Stream Ship quality judgments with an explicit 64k response contract

* Validate compact judge reasoning locally with supported wire schema

* Align functional QA fixture instructions with evidence acceptance

* Bind native browser diagnostics to execution evidence and align review verdicts

* Preserve native diagnostic line boundaries

* Serialize functional QA evidence from native captures

* Keep large QA evidence fixture payload out of Windows argv
This commit is contained in:
Garry Tan authored and GitHub committed 2026-09-29 06:07:35 -07:00
1 parent 65bfb0ce49
commit dcaea52800
333 files changed
+41755 -7357
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

test/qa-only-capability.test.ts
+49 -7
View File
@@ -11,7 +11,7 @@ test('QA-only capability regressions select the no-fix case', () => {
expect(selectTests(['test/qa-only-capability.test.ts'], E2E_TOUCHFILES).selected).toEqual(['qa-only-no-fix']);
});
test.each(['success', 'omitted-tools', 'report-edit', 'source-edit', 'source-write'])
test.each(['success', 'omitted-tools', 'report-edit', 'source-edit', 'source-write', 'preparation-late', 'charter-sidecar', 'memory-write', 'checkpoint-rewrite'])
('QA-only registered capability and no-fix contract: %s', scenario => {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'qa-tools-'));
const bin = path.join(dir, 'bin');
@@ -20,16 +20,45 @@ test.each(['success', 'omitted-tools', 'report-edit', 'source-edit', 'source-wri
const facts = path.join(dir, 'facts.json');
fs.writeFileSync(path.join(bin, 'claude'), `#!${process.execPath}
const fs = require('node:fs'), path = require('node:path');
const {spawnSync} = require('node:child_process');
await Bun.stdin.text();
const args = process.argv.slice(2), at = args.indexOf('--tools');
fs.writeFileSync(${JSON.stringify(facts)}, JSON.stringify({args}));
const scenario = ${JSON.stringify(scenario)};
const report = path.join(process.cwd(), 'qa-reports/qa-only-report.md');
fs.mkdirSync(path.dirname(report), {recursive:true});
fs.writeFileSync(report, '| **Total** | **7** |');
const emit = (name,input) => console.log(JSON.stringify({type:'assistant',message:{content:[{type:'tool_use',id:'fixture-'+name,name,input}]}}));
let callId=0;
const emit = (name,input,output) => {
const id='fixture-'+(++callId);
console.log(JSON.stringify({type:'assistant',message:{content:[{type:'tool_use',id,name,input}]}}));
if(output!==undefined)console.log(JSON.stringify({type:'user',message:{content:[{type:'tool_result',tool_use_id:id,content:output}]}}));
};
console.log(JSON.stringify({type:'system',subtype:'init',tools:at < 0 ? ['Bash','Read','Write','Glob','Edit'] : args[at+1].split(',')}));
emit('Write',{file_path:report,content:'| **Total** | **7** |'});
const writeReport=(target=report)=>{
fs.writeFileSync(target,'| **Total** | **7** |');
emit('Write',{file_path:target,content:'| **Total** | **7** |'},'Write completed');
};
if(scenario!=='preparation-late')writeReport(scenario==='charter-sidecar'?path.join(path.dirname(report),'charters.md'):report);
const guard=${JSON.stringify(path.join(ROOT, 'bin/gstack-qa-deadline'))}, deadline=path.join(process.cwd(),'qa-reports/deadline.json');
let lastCommand='';
for(const args of [['start',deadline,'30'],['run',deadline,'--',process.execPath,'--version']]){
const child=spawnSync(process.execPath,[guard,...args],{encoding:'utf8',timeout:5000});
if(child.error||child.status!==0)throw child.error||new Error(child.stderr);
lastCommand=[process.execPath,guard,...args].map(arg=>JSON.stringify(arg)).join(' ');
emit('Bash',{command:lastCommand},child.stdout+child.stderr);
}
if(scenario==='checkpoint-rewrite'){
const target=path.join(path.dirname(report),'exploration-001.json');
const content=JSON.stringify({observationCommand:lastCommand,observed:'Rewritten child output',hypothesis:'Check again',nextCommand:lastCommand});
fs.writeFileSync(target,content);emit('Write',{file_path:target,content},'Write completed');
}
if(scenario==='preparation-late'||scenario==='charter-sidecar')writeReport();
if(scenario==='memory-write'){
const target=path.join(process.env.HOME,'memory','MEMORY.md');
fs.mkdirSync(path.dirname(target),{recursive:true});
fs.writeFileSync(target,'An operational learning.');
emit('Write',{file_path:target,content:'An operational learning.'},'Write completed');
}
if(scenario === 'report-edit' || scenario === 'source-edit') {
const target = scenario === 'report-edit' ? report : path.join(process.cwd(),'index.html');
const old_string = fs.readFileSync(target,'utf8');
@@ -67,8 +96,10 @@ mock.module(path.join(root,'test/helpers/e2e-helpers.ts'),()=>({
mock.module(path.join(root,'test/helpers/session-runner.ts'),()=>({...actual,runSkillTest:async options=>{
const tools=['Bash','Read','Write','Glob'];
expect(options.allowedTools).toEqual(tools);expect(options.tools).toEqual(tools);
expect(options.maxTurns).toBe(40);expect(options.timeout).toBe(300000);
expect(options.maxTurns).toBe(40);expect(options.timeout).toBeGreaterThan(0);expect(options.timeout).toBeLessThanOrEqual(300000);
expect(options.prompt).toContain('Write your report to '+options.workingDirectory+'/qa-reports/qa-only-report.md');
expect(options.prompt).toContain('Memory files and learning stores outside that directory are not authorized');
expect(options.prompt).toContain('Write the initial charters and final report to the same caller-owned file '+options.workingDirectory+'/qa-reports/qa-only-report.md');
const launch={...options,timeout:2000,runId:undefined};
if(scenario==='omitted-tools')delete launch.tools;
const result=await runActual(launch);
@@ -102,8 +133,19 @@ await import(path.join(root,'test/skill-e2e-qa-workflow.test.ts'));
expect(observed.args[observed.args.indexOf('--tools') + 1]).toBe('Bash,Read,Write,Glob');
const editCount = observed.calls.filter((call: {tool:string}) => call.tool === 'Edit').length;
expect(editCount).toBe(scenario.endsWith('-edit') ? 1 : 0);
if (scenario !== 'source-write') expect(observed.recorded.passed).toBe(!scenario.endsWith('-edit'));
if (scenario !== 'success') expect(child.stderr).toContain('toHaveLength(0)');
if (scenario === 'preparation-late' || scenario === 'charter-sidecar') {
expect(observed.recorded.passed).toBe(false);
expect(child.stderr).toContain('QA preparation:');
} else if (scenario === 'memory-write') {
expect(observed.recorded.passed).toBe(false);
expect(child.stderr).toContain('QA deadline: artifact outside owned directory');
} else if (scenario === 'checkpoint-rewrite') {
expect(observed.recorded.passed).toBe(false);
expect(child.stderr).toContain('QA checkpoint: observation differs');
} else {
if (scenario !== 'source-write') expect(observed.recorded.passed).toBe(!scenario.endsWith('-edit'));
if (scenario !== 'success') expect(child.stderr).toContain('toHaveLength(0)');
}
}
} finally {
fs.rmSync(dir, {recursive:true,force:true});