fix(flaky): qa-only diagnostic reads browse-daemon.log; router cases score a long session; retention and welcome tests name their cause (GST-1632) (#4074)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

browse/test/xvfb.test.ts

@@ -471,7 +471,10 @@ describe.skipIf(process.platform !== 'linux')('daemon-owned display lifecycle',
           if (!response.ok) throw new Error(`${name}: ${response.status}: ${await response.text()}`);
           return response.text();
         };
-        if (mode === 'welcome') expect(await command('url')).toContain('/welcome');
+        if (mode === 'welcome') {
+          const url = await command('url');
+          expect(url, `daemon log tail: ${fs.readFileSync(path.join(root, 'daemon.log'), 'utf8').slice(-800)}`).toContain('/welcome');
+        }
         if (mode === 'welcome failure') {
           expect(fs.readFileSync(path.join(root, 'daemon.log'), 'utf8')).toContain('Welcome test navigation failure');
           expect(await command('url')).toContain('about:blank');

changelog.d/gsta348-flaky.md

@@ -0,0 +1 @@
+- **Flaky pass 2026-11-06: four flaky tests now report why they failed, and one paid case is scored instead of killed.** The QA-only cleanup diagnostic reads `browse-daemon.log`, the file a CLI-spawned browse daemon writes, so a missing terminal record now shows the daemon's terminal-agent lines instead of "none". The router failure-routing cases give their session a deadline 10 seconds inside the test's own timeout, so a session that runs long is scored on the routing it already did rather than lost to Bun's kill (paid census 42103721891: routed to /investigate after two reads, then a 300-second stress command ran out the clock). The bootstrap retention test prints the paid shard's outcome and log lines when its status is wrong, and the Xvfb welcome test prints the daemon log when the page is not the welcome page. No threshold, trial count, case kind or model budget changed. (GST-1632)

test/bootstrap-retention-shard.test.ts

@@ -50,7 +50,7 @@ describe.skipIf(process.platform !== 'linux')('bootstrap paid-shard cleanup inte
         import { runPaidShard } from ${JSON.stringify(path.join(import.meta.dir, '../scripts/test-paid-shards.ts'))};
         const outcome = await runPaidShard(['test/skill-e2e-qa-workflow.test.ts'], 1, 1, {
           rootDir: ${JSON.stringify(path.join(import.meta.dir, '..'))}, timeoutMs: 12000, jobs: 2, logDir: ${JSON.stringify(root)},
-          evalDirBase: ${JSON.stringify(path.join(root, 'artifacts'))}, log: () => {},
+          evalDirBase: ${JSON.stringify(path.join(root, 'artifacts'))}, log: line => console.log(line),
           env: { PATH: process.env.PATH, GSTACK_CLAUDE_CLI_VERSION: 'synthetic-no-provider', EVALS_RUN_ID: 'integration-run' },
           commandFor: () => ({ command: process.execPath, args: ['test', ${JSON.stringify(script)}, '--retry', ${JSON.stringify(scenario === 'retry' ? '1' : '0')}, '--timeout', '10000'] }),
         });
@@ -71,7 +71,7 @@ describe.skipIf(process.platform !== 'linux')('bootstrap paid-shard cleanup inte
       const attempts = fs.readFileSync(path.join(root, 'attempts.jsonl'), 'utf8').trim().split('\n').map(line => JSON.parse(line));
       expect(attempts.length).toBe(scenario === 'retry' ? 2 : 1);
       expect(new Set(attempts.map(attempt => attempt.artifact)).size).toBe(attempts.length);
-      expect(outcome.status).toBe(scenario === 'success' || scenario === 'retry' ? 'passed' : 'failed');
+      expect(outcome.status, `shard ${JSON.stringify({ ...outcome, files: undefined })}; controller stdout tail: ${(controller.stdout ?? '').slice(-1200).replace(/\s*\n\s*/g, ' | ')}`).toBe(scenario === 'success' || scenario === 'retry' ? 'passed' : 'failed');
       for (const attempt of attempts) {
         const state = path.dirname(path.dirname(attempt.root));
         if (scenario === 'ack-failure') {

test/helpers/qa-only-cleanup.ts

@@ -55,7 +55,7 @@ async function settleOwnedBrowser(directory: string, deadlineEpochMs: number): P
   const agent = readAgentRecord(stateDir);
   if (!agentRaw || !agent) {
     // Qa-only-no-fix (census 41962523474) failed here once with nothing else to go on; name what was there instead.
-    const log = path.join(stateDir, 'browse-server.log');
+    const log = path.join(stateDir, 'browse-daemon.log');
     const agentLines = fs.existsSync(log) && fs.lstatSync(log).isFile()
       ? fs.readFileSync(log, 'utf8').split('\n').filter(line => /terminal-agent/i.test(line)).slice(-3).map(line => line.slice(0, 160)) : [];
     fail(`terminal identity is unavailable (record ${agentRaw === undefined ? 'absent' : `unparsed, ${agentRaw.length} bytes`}; `

test/qa-only-cleanup.test.ts

@@ -87,7 +87,10 @@ fs.writeFileSync(${JSON.stringify(path.join(dir, 'port'))}, String(server.port))
       chromiumPid, chromiumStartTime: scenario === 'chromium-mismatch' ? 'stale start' : readPidStartTime(chromiumPid) }));
     const agentFile = path.join(stateDir, 'terminal-agent-pid');
     if (scenario === 'terminal-mismatch') fs.writeFileSync(agentFile, JSON.stringify({ ...agent, ownerPid: process.pid }));
-    if (scenario === 'missing-terminal') fs.unlinkSync(agentFile);
+    if (scenario === 'missing-terminal') {
+      fs.unlinkSync(agentFile);
+      fs.writeFileSync(path.join(stateDir, 'browse-daemon.log'), '[browse] terminal-agent exit remains unconfirmed: fixture\n');
+    }
     if (scenario === 'blocked-identity') {
       const preload = path.join(dir, 'block-identity.ts');
       fs.writeFileSync(preload, `
@@ -113,7 +116,7 @@ Bun.spawnSync = (...args) => {
     const started = performance.now();
     const errors = { 'ack-only': 'settlement deadline exceeded', 'replaced-state': 'daemon state was replaced',
       'terminal-mismatch': 'terminal ownership is unconfirmed', 'chromium-mismatch': 'Chromium ownership is unconfirmed',
-      'missing-terminal': 'terminal identity is unavailable (record absent; terminal-port', 'blocked-identity': 'owned worker settlement deadline exceeded' };
+      'missing-terminal': /terminal identity is unavailable \(record absent; terminal-port .*daemon log terminal-agent lines: \[browse\] terminal-agent exit remains unconfirmed: fixture\)/, 'blocked-identity': 'owned worker settlement deadline exceeded' };
     await expect(stopQaOnlyBrowser(dir, 1000)).rejects.toThrow(errors[scenario]);
     expect(performance.now() - started).toBeLessThan(2000);
     expect(daemon.exitCode).toBeNull();

test/skill-e2e-failure-routing.test.ts

@@ -31,6 +31,7 @@ import { recordObservation } from './helpers/none-arm';
 
 const collector = createEvalCollector('e2e-failure-routing');
 const ARM = routerArm();
+const SCORE_GRACE_MS = 10_000;
 
 describeIfSelected('Router failure routing (#3157, #3158)', FIXTURES.map(f => f.name), () => {
   const dirs: string[] = [];
@@ -50,7 +51,7 @@ describeIfSelected('Router failure routing (#3157, #3158)', FIXTURES.map(f => f.
         maxTurns: 6,
         tools: ['Skill', 'Read', 'Grep', 'Glob', 'Edit', 'Bash'],
         allowedTools: ['Skill', 'Read', 'Grep', 'Glob', 'Edit', 'Bash'],
-        timeout: CAPTURE_MS,
+        timeout: CAPTURE_MS - SCORE_GRACE_MS,
         testName: fixture.name,
         runId,
       });