diff --git a/browse/test/xvfb.test.ts b/browse/test/xvfb.test.ts index 67f305a66..595bfa9ed 100644 --- a/browse/test/xvfb.test.ts +++ b/browse/test/xvfb.test.ts @@ -471,7 +471,10 @@ describe.skipIf(process.platform !== 'linux')('daemon-owned display lifecycle', if (!response.ok) throw new Error(`${name}: ${response.status}: ${await response.text()}`); return response.text(); }; - if (mode === 'welcome') expect(await command('url')).toContain('/welcome'); + if (mode === 'welcome') { + const url = await command('url'); + expect(url, `daemon log tail: ${fs.readFileSync(path.join(root, 'daemon.log'), 'utf8').slice(-800)}`).toContain('/welcome'); + } if (mode === 'welcome failure') { expect(fs.readFileSync(path.join(root, 'daemon.log'), 'utf8')).toContain('Welcome test navigation failure'); expect(await command('url')).toContain('about:blank'); diff --git a/changelog.d/gsta348-flaky.md b/changelog.d/gsta348-flaky.md new file mode 100644 index 000000000..0814ce2d5 --- /dev/null +++ b/changelog.d/gsta348-flaky.md @@ -0,0 +1 @@ +- **Flaky pass 2026-11-06: four flaky tests now report why they failed, and one paid case is scored instead of killed.** The QA-only cleanup diagnostic reads `browse-daemon.log`, the file a CLI-spawned browse daemon writes, so a missing terminal record now shows the daemon's terminal-agent lines instead of "none". The router failure-routing cases give their session a deadline 10 seconds inside the test's own timeout, so a session that runs long is scored on the routing it already did rather than lost to Bun's kill (paid census 42103721891: routed to /investigate after two reads, then a 300-second stress command ran out the clock). The bootstrap retention test prints the paid shard's outcome and log lines when its status is wrong, and the Xvfb welcome test prints the daemon log when the page is not the welcome page. No threshold, trial count, case kind or model budget changed. (GST-1632) diff --git a/test/bootstrap-retention-shard.test.ts b/test/bootstrap-retention-shard.test.ts index 2b120bd4e..2340991bf 100644 --- a/test/bootstrap-retention-shard.test.ts +++ b/test/bootstrap-retention-shard.test.ts @@ -50,7 +50,7 @@ describe.skipIf(process.platform !== 'linux')('bootstrap paid-shard cleanup inte import { runPaidShard } from ${JSON.stringify(path.join(import.meta.dir, '../scripts/test-paid-shards.ts'))}; const outcome = await runPaidShard(['test/skill-e2e-qa-workflow.test.ts'], 1, 1, { rootDir: ${JSON.stringify(path.join(import.meta.dir, '..'))}, timeoutMs: 12000, jobs: 2, logDir: ${JSON.stringify(root)}, - evalDirBase: ${JSON.stringify(path.join(root, 'artifacts'))}, log: () => {}, + evalDirBase: ${JSON.stringify(path.join(root, 'artifacts'))}, log: line => console.log(line), env: { PATH: process.env.PATH, GSTACK_CLAUDE_CLI_VERSION: 'synthetic-no-provider', EVALS_RUN_ID: 'integration-run' }, commandFor: () => ({ command: process.execPath, args: ['test', ${JSON.stringify(script)}, '--retry', ${JSON.stringify(scenario === 'retry' ? '1' : '0')}, '--timeout', '10000'] }), }); @@ -71,7 +71,7 @@ describe.skipIf(process.platform !== 'linux')('bootstrap paid-shard cleanup inte const attempts = fs.readFileSync(path.join(root, 'attempts.jsonl'), 'utf8').trim().split('\n').map(line => JSON.parse(line)); expect(attempts.length).toBe(scenario === 'retry' ? 2 : 1); expect(new Set(attempts.map(attempt => attempt.artifact)).size).toBe(attempts.length); - expect(outcome.status).toBe(scenario === 'success' || scenario === 'retry' ? 'passed' : 'failed'); + expect(outcome.status, `shard ${JSON.stringify({ ...outcome, files: undefined })}; controller stdout tail: ${(controller.stdout ?? '').slice(-1200).replace(/\s*\n\s*/g, ' | ')}`).toBe(scenario === 'success' || scenario === 'retry' ? 'passed' : 'failed'); for (const attempt of attempts) { const state = path.dirname(path.dirname(attempt.root)); if (scenario === 'ack-failure') { diff --git a/test/helpers/qa-only-cleanup.ts b/test/helpers/qa-only-cleanup.ts index f255faf45..be6a69243 100644 --- a/test/helpers/qa-only-cleanup.ts +++ b/test/helpers/qa-only-cleanup.ts @@ -55,7 +55,7 @@ async function settleOwnedBrowser(directory: string, deadlineEpochMs: number): P const agent = readAgentRecord(stateDir); if (!agentRaw || !agent) { // Qa-only-no-fix (census 41962523474) failed here once with nothing else to go on; name what was there instead. - const log = path.join(stateDir, 'browse-server.log'); + const log = path.join(stateDir, 'browse-daemon.log'); const agentLines = fs.existsSync(log) && fs.lstatSync(log).isFile() ? fs.readFileSync(log, 'utf8').split('\n').filter(line => /terminal-agent/i.test(line)).slice(-3).map(line => line.slice(0, 160)) : []; fail(`terminal identity is unavailable (record ${agentRaw === undefined ? 'absent' : `unparsed, ${agentRaw.length} bytes`}; ` diff --git a/test/qa-only-cleanup.test.ts b/test/qa-only-cleanup.test.ts index f834ff9cf..3ee38b900 100644 --- a/test/qa-only-cleanup.test.ts +++ b/test/qa-only-cleanup.test.ts @@ -87,7 +87,10 @@ fs.writeFileSync(${JSON.stringify(path.join(dir, 'port'))}, String(server.port)) chromiumPid, chromiumStartTime: scenario === 'chromium-mismatch' ? 'stale start' : readPidStartTime(chromiumPid) })); const agentFile = path.join(stateDir, 'terminal-agent-pid'); if (scenario === 'terminal-mismatch') fs.writeFileSync(agentFile, JSON.stringify({ ...agent, ownerPid: process.pid })); - if (scenario === 'missing-terminal') fs.unlinkSync(agentFile); + if (scenario === 'missing-terminal') { + fs.unlinkSync(agentFile); + fs.writeFileSync(path.join(stateDir, 'browse-daemon.log'), '[browse] terminal-agent exit remains unconfirmed: fixture\n'); + } if (scenario === 'blocked-identity') { const preload = path.join(dir, 'block-identity.ts'); fs.writeFileSync(preload, ` @@ -113,7 +116,7 @@ Bun.spawnSync = (...args) => { const started = performance.now(); const errors = { 'ack-only': 'settlement deadline exceeded', 'replaced-state': 'daemon state was replaced', 'terminal-mismatch': 'terminal ownership is unconfirmed', 'chromium-mismatch': 'Chromium ownership is unconfirmed', - 'missing-terminal': 'terminal identity is unavailable (record absent; terminal-port', 'blocked-identity': 'owned worker settlement deadline exceeded' }; + 'missing-terminal': /terminal identity is unavailable \(record absent; terminal-port .*daemon log terminal-agent lines: \[browse\] terminal-agent exit remains unconfirmed: fixture\)/, 'blocked-identity': 'owned worker settlement deadline exceeded' }; await expect(stopQaOnlyBrowser(dir, 1000)).rejects.toThrow(errors[scenario]); expect(performance.now() - started).toBeLessThan(2000); expect(daemon.exitCode).toBeNull(); diff --git a/test/skill-e2e-failure-routing.test.ts b/test/skill-e2e-failure-routing.test.ts index 6cfa04781..a4e28aca8 100644 --- a/test/skill-e2e-failure-routing.test.ts +++ b/test/skill-e2e-failure-routing.test.ts @@ -31,6 +31,7 @@ import { recordObservation } from './helpers/none-arm'; const collector = createEvalCollector('e2e-failure-routing'); const ARM = routerArm(); +const SCORE_GRACE_MS = 10_000; describeIfSelected('Router failure routing (#3157, #3158)', FIXTURES.map(f => f.name), () => { const dirs: string[] = []; @@ -50,7 +51,7 @@ describeIfSelected('Router failure routing (#3157, #3158)', FIXTURES.map(f => f. maxTurns: 6, tools: ['Skill', 'Read', 'Grep', 'Glob', 'Edit', 'Bash'], allowedTools: ['Skill', 'Read', 'Grep', 'Glob', 'Edit', 'Bash'], - timeout: CAPTURE_MS, + timeout: CAPTURE_MS - SCORE_GRACE_MS, testName: fixture.name, runId, });