mirror of
https://github.com/garrytan/gstack.git
synced 2026-10-03 01:46:55 +02:00
feat(evals): stamp trial series identities and fit panels to the live registry
- scripts/eval-trial-series.ts stamps series_identity (eval-flake-rank's caseSeriesIdentities) on a report's trial-outcomes JSONL as its own step, keeping the history tool out of the paid runner's closure; TrialOutcomeRecord gains the optional series_identity field. - Slice-count plans let a registered trial spill into an ordinary lane when its siblings hold every long lane, so panels never share a runner. - Re-audited test-selection.ts (Stream B added the E2E_KINDS/BEHAVIOR_WHY map-diff; no new module loading) and repinned its hash. - Detach and release floors now count trial shards (66 periodic trials in 22 panels): periodic floor 33,821s, still under eval:bg:periodic's 67,380s. - Coordination fixtures supply the executor's trial records.
This commit is contained in:
1 parent
d5876efa5d
commit
62fb9a255d
8 files changed
+76
-31
No files matched your search
@@ -0,0 +1,35 @@
|
||||
#!/usr/bin/env bun
|
||||
/**
|
||||
* Stamp `series_identity` on a report's trial-outcomes JSONL (the pass-rates
|
||||
* history key: a hash of each case's own touchfiles, GLOBAL_TOUCHFILES
|
||||
* excluded; scripts/eval-flake-rank.ts caseSeriesIdentities). A separate step
|
||||
* after `test-paid-shards.ts --report`, so the paid runner's closure never
|
||||
* imports the history tool.
|
||||
*
|
||||
* Usage: bun run scripts/eval-trial-series.ts <trial-outcomes.jsonl>
|
||||
*/
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { caseSeriesIdentities } from './eval-flake-rank';
|
||||
import { formatTrialOutcomes, parseTrialOutcomes } from '../test/helpers/eval-store';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
|
||||
/** Rewrite the file with every record stamped; an invalid line fails the whole stamp. */
|
||||
export function stampTrialSeries(file: string, root = ROOT): number {
|
||||
const { records, errors } = parseTrialOutcomes(fs.readFileSync(file, 'utf8'));
|
||||
if (errors.length) throw new Error(`${file}: ${errors.join('; ')}`);
|
||||
const identities = caseSeriesIdentities([...new Set(records.map(record => record.case))], root);
|
||||
const stamped = records.map(record => ({ ...record, series_identity: identities[record.case] }));
|
||||
fs.writeFileSync(file, formatTrialOutcomes(stamped));
|
||||
return stamped.length;
|
||||
}
|
||||
|
||||
if (import.meta.main) {
|
||||
const file = process.argv[2];
|
||||
if (!file) {
|
||||
console.error('usage: bun run scripts/eval-trial-series.ts <trial-outcomes.jsonl>');
|
||||
process.exit(2);
|
||||
}
|
||||
console.log(`[eval-trial-series] stamped ${stampTrialSeries(file)} record(s) in ${file}`);
|
||||
}
|
||||
Reference in new issue
Block a user