mirror of
https://github.com/garrytan/gstack.git
synced 2026-10-04 02:16:56 +02:00
fix(evals): --case list mode and name precheck; case-shard qa-callers; refresh batching and design-with-ui seeds
This commit is contained in:
1 parent
2dae4bf944
commit
581e603313
6 files changed
+41
-14
No files matched your search
@@ -21,7 +21,7 @@
|
||||
* Failure taxonomy (CEO review finding 2):
|
||||
* - zero-diff arm -> VALID scored cell (LOC 0, judge scores it "none").
|
||||
* - harvest failure -> cell FAILED, harvest: null recorded.
|
||||
* - judge still malformed after armJudge's bounded retries -> judge_error
|
||||
* - judge output malformed (armJudge never re-asks) -> judge_error
|
||||
* cell: excluded from aggregates, surfaced in the run report, never
|
||||
* silently dropped.
|
||||
*
|
||||
@@ -108,7 +108,7 @@ async function runArmCell(task: ArmTask, arm: Arm): Promise<CellResult> {
|
||||
}
|
||||
const checks = runChecks(task, dirs.dir);
|
||||
|
||||
// Judge taxonomy: still malformed after armJudge's bounded retries ->
|
||||
// Judge taxonomy: malformed armJudge output (no re-ask) ->
|
||||
// judge_error cell (excluded from aggregates, surfaced in the report).
|
||||
let judge: ArmJudgeScore | null = null;
|
||||
let judgeError: string | null = null;
|
||||
|
||||
Reference in new issue
Block a user