fix: adversarial-review runtime fixes across the telemetry + kill paths

- session-runner: exit-labeling keys off 'exit', not 'close' — an orphan
  holding the pipes could relabel a REAL exit (auth failure) as
  'timeout_startup' availability noise; the kill path still always
  group-kills and cancels the reader (labeling and unblocking are separate
  concerns). Work phase arms on a flag, not firstResponseMs===0 (a same-ms
  first byte left the startup timer live all run). The CI startup grace is
  now a real FLOOR (Math.max), matching its name and pinning test.
- gstack-detach: pgid captured AT SPAWN (== child pid under
  start_new_session) — resolving it after the grace raised ESRCH once the
  leader died on SIGTERM, orphaning TERM-immune grandchildren forever.
- test-free-shards: ledger entries carry branch + git_sha (rev-parse split:
  '--abbrev-ref HEAD HEAD' printed the branch twice and recorded it as the
  sha); local ledger default is per-PROJECT, not the machine-global tmpdir.
- eval-flake-rank: per-LINE ledger parse (one torn JSONL line vanished the
  whole series), 60-day recency bound (transcript-bearing files are MBs),
  shared isFinalizedEvalResultFile predicate (the artifact-taxonomy rule
  lived in three places); eval-store exports the predicate and finalize
  stops computing flakyRetries twice; paid-shards cleanup uses async rm
  (a SIGKILLed shard's git-workspace teardown blocked every sibling's
  stream classification on the parent event loop).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Garry Tan
2026-08-31 05:38:26 +00:00
co-authored by Claude Fable 5
parent 5b04f05ba4
commit 513111a166
6 changed files with 116 additions and 21 deletions
+24 -7
View File
@@ -21,7 +21,7 @@
import * as fs from 'node:fs';
import * as path from 'node:path';
import { getProjectEvalDir, isPartialEval, type EvalResult } from '../test/helpers/eval-store';
import { getProjectEvalDir, isPartialEval, isFinalizedEvalResultFile, type EvalResult } from '../test/helpers/eval-store';
import { flakeLedgerPath, type FlakeLedgerEntry } from './test-free-shards';
interface TestSeries {
@@ -73,21 +73,36 @@ export function aggregate(evalFiles: string[]): Map<string, TestSeries> {
return series;
}
export function collectEvalFiles(dir: string): string[] {
export function collectEvalFiles(dir: string, sinceDays = 60): string[] {
if (!fs.existsSync(dir)) return [];
const cutoff = Date.now() - sinceDays * 86_400_000;
const out: string[] = [];
for (const name of fs.readdirSync(dir, { recursive: true }) as string[]) {
if (!name.endsWith('.json') || path.basename(name).startsWith('_partial')) continue;
out.push(path.join(dir, name));
if (!isFinalizedEvalResultFile(name)) continue;
const full = path.join(dir, name);
try {
// Recency bound (review finding): E2E results embed full transcripts
// (MBs each) and the scan is otherwise unbounded over all-time history.
if (fs.statSync(full).mtimeMs < cutoff) continue;
} catch { continue; }
out.push(full);
}
return out;
}
function readFreeLedger(): FlakeLedgerEntry[] {
// Per-LINE parse: one malformed JSONL line (torn write, manual edit) must
// drop that line, never vanish the whole series (codex adversarial finding).
let raw: string;
try {
return fs.readFileSync(flakeLedgerPath(), 'utf-8')
.split('\n').filter(Boolean).map((l) => JSON.parse(l));
raw = fs.readFileSync(flakeLedgerPath(), 'utf-8');
} catch { return []; }
const out: FlakeLedgerEntry[] = [];
for (const line of raw.split('\n')) {
if (!line.trim()) continue;
try { out.push(JSON.parse(line)); } catch { /* torn line — skip */ }
}
return out;
}
if (import.meta.main) {
@@ -95,8 +110,10 @@ if (import.meta.main) {
const dirFlag = argv.indexOf('--dir');
const dir = dirFlag !== -1 ? argv[dirFlag + 1] : getProjectEvalDir();
const asJson = argv.includes('--json');
const sinceFlag = argv.indexOf('--since-days');
const sinceDays = sinceFlag !== -1 ? Number(argv[sinceFlag + 1]) || 60 : 60;
const files = collectEvalFiles(dir);
const files = collectEvalFiles(dir, sinceDays);
const series = [...aggregate(files).values()]
.sort((a, b) => b.retriedPasses - a.retriedPasses || (b.fails / b.runs) - (a.fails / a.runs));
const ledger = readFreeLedger();