fix(evals): --case list mode and name precheck; case-shard qa-callers; refresh batching and design-with-ui seeds

This commit is contained in:
garrytan committed 2026-09-29 20:28:31 +00:00
1 parent 2dae4bf944
commit 581e603313
6 files changed
+41 -14

No files matched your search

+2 -2
View File
@@ -21,7 +21,7 @@
* Failure taxonomy (CEO review finding 2):
* - zero-diff arm -> VALID scored cell (LOC 0, judge scores it "none").
* - harvest failure -> cell FAILED, harvest: null recorded.
* - judge still malformed after armJudge's bounded retries -> judge_error
* - judge output malformed (armJudge never re-asks) -> judge_error
* cell: excluded from aggregates, surfaced in the run report, never
* silently dropped.
*
@@ -108,7 +108,7 @@ async function runArmCell(task: ArmTask, arm: Arm): Promise<CellResult> {
}
const checks = runChecks(task, dirs.dir);
// Judge taxonomy: still malformed after armJudge's bounded retries ->
// Judge taxonomy: malformed armJudge output (no re-ask) ->
// judge_error cell (excluded from aggregates, surfaced in the report).
let judge: ArmJudgeScore | null = null;
let judgeError: string | null = null;