diff --git a/changelog.d/gsta262-vm-family.md b/changelog.d/gsta262-vm-family.md new file mode 100644 index 000000000..8da455d59 --- /dev/null +++ b/changelog.d/gsta262-vm-family.md @@ -0,0 +1,5 @@ +### Tests +- Three free tests that failed only on a busy remote test machine (remote pregate run 20261103T015240Z-11756-6ff6, PR #3892) now hold up under load: + - `test/review-army-budget.test.ts`: the red-team retry fixture spent a real 80 ms sleep inside its 200 ms deadline, so a pause of more than 120 ms failed attempt 2. Attempt 1 now waits for the cancellation and then drains, and attempt 2 doesn't sleep. With random 150 ms pauses injected, 6 of 10 runs passed before and 20 of 20 pass now. + - `test/helpers/docsync-fault-callbacks.ts`: the docsync replay assumed fixture setup took less than 5 seconds, and setup takes 5 to 12 seconds on a busy machine. The replay now checks that the session timeout equals the budget minus 15 seconds minus the time already spent, which is how the code computes it. Under 16 busy loops on 4 cores, 0 of 4 runs passed before and 4 of 4 pass now. + - `test/fixtures/plan-seed-cli.ts`: the fake CLI drew its screen and its startup question in two separate writes, so a screen read between them saw an empty input and pasted the plan. It now draws both in one write. Under load, 31 of 36 runs passed before and 36 of 36 pass now. diff --git a/test/fixtures/plan-seed-cli.ts b/test/fixtures/plan-seed-cli.ts index 9323d5749..f740abe21 100644 --- a/test/fixtures/plan-seed-cli.ts +++ b/test/fixtures/plan-seed-cli.ts @@ -22,10 +22,10 @@ const row=(type,content,stop)=>JSON.stringify({type,sessionId:sid,cwd,message:{r const text=s=>[{type:'text',text:s}]; const append=(type,content,stop)=>fs.appendFileSync(file,row(type,content,stop)); const rule='─'.repeat(120); -const frame=(s,history='',top=rule,bottom=rule)=>{ +const frame=(s,history='',top=rule,bottom=rule,tail='')=>{ const inputRows=1+(s.match(/\r\n/g)||[]).length; const topRow=(process.stdout.rows||40)-inputRows-2; - process.stdout.write('\x1b[2J\x1b[H'+history+'\x1b['+topRow+';1H'+top+'\r\n❯ '+s+'\r\n'+bottom+'\r\npaste again to expand'); + process.stdout.write('\x1b[2J\x1b[H'+history+'\x1b['+topRow+';1H'+top+'\r\n❯ '+s+'\r\n'+bottom+'\r\npaste again to expand'+tail); }; let input='',seed='',submitted=false; process.stdin.setRawMode(true);process.stdin.resume(); @@ -40,12 +40,12 @@ else if(scenario==='startup-ci-placeholder')frame(process.env.CI==='true'&&proce else if(scenario==='startup-ci-typed-hint')frame(hint); else if(scenario==='startup-placeholder-cursor')frame('\x1b[7mT\x1b[27m\x1b[2m'+hint.slice(1)+'\x1b[22m'); else if(scenario==='startup-placeholder-unicode')frame('\x1b[2mTry "refactor src/設定.ts"\x1b[22m'); -else if(['startup-placeholder','startup-prior-conversation','startup-missing-styles','startup-waiting','startup-prose-question','startup-permission','startup-fresh-waiting'].includes(scenario))frame('\x1b[2m'+hint+'\x1b[22m'); +else if(['startup-placeholder','startup-prior-conversation','startup-missing-styles','startup-waiting','startup-prose-question','startup-permission','startup-fresh-waiting'].includes(scenario))frame('\x1b[2m'+hint+'\x1b[22m','',rule,rule, + scenario==='startup-prose-question'?'Which option do you prefer?\r\nA) Full review (recommended)\r\nB) Skip review\r\n' + :scenario==='startup-permission'?'Bash command run checks requires permission\r\n':''); else if(scenario==='startup-typed-hint')frame(hint); else if(scenario==='startup-partial-dim')frame('\x1b[2mTry \x1b[22m"refactor "'); else frame(''); -if(scenario==='startup-prose-question')process.stdout.write('Which option do you prefer?\r\nA) Full review (recommended)\r\nB) Skip review\r\n'); -if(scenario==='startup-permission')process.stdout.write('Bash command run checks requires permission\r\n'); process.stdin.on('data',chunk=>{ input+=chunk.toString(); if(input.startsWith('\x1b[200~')&&input.endsWith('\x1b[201~')){ diff --git a/test/helpers/docsync-fault-callbacks.ts b/test/helpers/docsync-fault-callbacks.ts index 31a158557..abdaab99e 100644 --- a/test/helpers/docsync-fault-callbacks.ts +++ b/test/helpers/docsync-fault-callbacks.ts @@ -25,7 +25,7 @@ const { parseDocsCompletion } = await import(path.join(root, 'test/helpers/docsy const { docsActorSeeded, DOCS_SEEDED_AUDIT_ID } = await import(path.join(root, 'test/helpers/docsync-fault-actor.ts')); const CALLBACK_TEST_MS = ${CALLBACK_TEST_MS}; const callbacks = new Map(); -let fixture, control = '', launches = 0, recorded, legacyReaudit = false; +let fixture, control = '', launches = 0, recorded, legacyReaudit = false, bodyStartedAt = 0; let returnedResult: SkillTestResult | undefined; let consumers: string[] = []; function expectResult(result: SkillTestResult) { @@ -47,7 +47,7 @@ mock.module(observerModule, () => ({ ...observers, mock.module(path.join(root, 'test/helpers/e2e-gate.ts'), () => ({ describeE2ETier: () => (_name, body) => body() })); mock.module(path.join(root, 'test/helpers/e2e-helpers.ts'), () => ({ runId: 'free-docsync-faults', describeIfSelected: (_name, _names, body) => body(), - testConcurrentIfSelected: (name, body, budget) => callbacks.set(name, { body, budget }), + testConcurrentIfSelected: (name, body, budget) => callbacks.set(name, { body: () => { bodyStartedAt = Date.now(); return body(); }, budget }), createEvalCollector: () => ({}), finalizeEvalCollector() {}, logCost(_name, result) { expectResult(result); @@ -65,8 +65,9 @@ mock.module(path.join(root, 'test/helpers/session-runner.ts'), () => ({ async ru const legacy = options.testName === 'ship-docsync-failure'; const scenario = legacy ? 'legacy-completion' : options.testName.slice('ship-docsync-'.length); expect(options.maxTurns).toBe(legacy ? 30 : 24); - expect(options.timeout).toBeGreaterThan((legacy ? CAPTURE_LONG_MS : CAPTURE_MS) - 20_000); - expect(options.timeout).toBeLessThanOrEqual((legacy ? CAPTURE_LONG_MS : CAPTURE_MS) - 15_000); + const remaining = (legacy ? CAPTURE_LONG_MS : CAPTURE_MS) - 15_000; + expect(options.timeout).toBeGreaterThanOrEqual(remaining - (Date.now() - bodyStartedAt)); + expect(options.timeout).toBeLessThanOrEqual(remaining); expect(options.tools).toEqual(['Bash', 'Read', 'Write', 'Edit', 'Glob', 'Grep']); expect(options.allowedTools).toEqual(options.tools); expect(options.prompt).toContain('BOUNDED_CALLBACK_USED'); diff --git a/test/review-army-budget.test.ts b/test/review-army-budget.test.ts index db3c5b808..9ba5d25ea 100644 --- a/test/review-army-budget.test.ts +++ b/test/review-army-budget.test.ts @@ -108,7 +108,10 @@ const run = async opts => { fixturePaths.push(opts.workingDirectory); captureOptions.push({ timeout: opts.timeout, maxTurns: opts.maxTurns, model: opts.model ?? null, signal: !!opts.signal }); opts.signal?.addEventListener('abort', () => events.push('abort-' + attempt), { once: true }); - await Bun.sleep(attempt === 1 ? captureMs + drainMs : 80); + if (attempt === 1) { + await new Promise(resolve => opts.signal.addEventListener('abort', resolve, { once: true })); + await Bun.sleep(drainMs); + } expect(fs.existsSync(opts.workingDirectory)).toBe(true); events.push('capture-done-' + attempt); if (attempt === 2) fs.writeFileSync(path.join(opts.workingDirectory, 'review-output.md'), ${JSON.stringify(validReport ? 'RED TEAM REVIEW' : 'ordinary unrelated prose')});