fix(evals): --case list mode and name precheck; case-shard qa-callers; refresh batching and design-with-ui seeds

test/cookie-validation-phases.test.ts

@@ -45,11 +45,12 @@ test('the existing quality and behavior phases retain their complete separate sh
   expect(behavior.evalsAll).toBe(true);
   expect(qualityFiles).toHaveLength(1);
   // 44 files (first-task-scaffold registers no gate case, so the gate lane
-  // skips it); the six case-sharded files contribute one shard per gate case.
+  // skips it); the seven case-sharded files contribute one shard per gate case.
   expect(new Set(behaviorFiles.map(file => file.split('#')[0])).size).toBe(44);
-  expect(behaviorFiles).toHaveLength(73);
+  expect(behaviorFiles).toHaveLength(77);
   expect(behaviorFiles).toEqual(expect.arrayContaining([
-    'test/skill-e2e-qa-callers.test.ts',
+    ...['review-exploratory-small-cli', 'ship-exploratory-small-cli', 'ship-exploratory-unavailable',
+      'ship-exploratory-plan-checks', 'ship-exploratory-late-input'].map(id => `test/skill-e2e-qa-callers.test.ts#${id}`),
     'test/skill-e2e-qa-functional-fix.test.ts',
     'test/skill-e2e-qa-functional.test.ts',
     'test/skill-e2e-ship-skip.test.ts',

test/paid-retry-supervision.test.ts

@@ -156,8 +156,8 @@ test('fixed AUQ count remains strict while mixed-tier files keep ordinary case h
     [AUQ_CONSISTENCY_RETRY_BUDGET.file, 0, false],
     [AUQ_CONSISTENCY_RETRY_BUDGET.file, 1, true],
     [AUQ_CONSISTENCY_RETRY_BUDGET.file, 2, false],
-    ['test/skill-e2e-qa-callers.test.ts', 3, true],
-    ['test/skill-e2e-qa-callers.test.ts', 5, true],
+    ['test/skill-e2e-plan-format.test.ts', 2, true],
+    ['test/skill-e2e-plan-format.test.ts', 4, true],
     // A case shard of a case-sharded registered file executes exactly its case.
     [plannedKey('test/skill-e2e-plan.test.ts'), 1, true],
     [plannedKey('test/skill-e2e-plan.test.ts'), 2, false],
@@ -211,7 +211,7 @@ test('detached PR fallback and release commands cover their actual default worke
   const prFloor = Math.ceil((Math.ceil(fullGateFiles.length / prWorkers) * 1_800_000 + fullGateFiles.reduce(
     (total, file) => total + Math.max(0, resolvePaidShardBudget([file]).timeoutMs - 1_800_000), 0,
   )) / 1000 * 1.05);
-  expect(prFloor).toBe(72_755);
+  expect(prFloor).toBe(76_535);
   expect(prWall).toBe(92_820_000);
   expect(prWall).toBeGreaterThanOrEqual(paidShardWallUpperBoundMs(files, prWorkers) + 120_000);
 

test/skill-e2e-arm-benchmark.test.ts

@@ -21,7 +21,7 @@
  * Failure taxonomy (CEO review finding 2):
  *   - zero-diff arm  -> VALID scored cell (LOC 0, judge scores it "none").
  *   - harvest failure -> cell FAILED, harvest: null recorded.
- *   - judge still malformed after armJudge's bounded retries -> judge_error
+ *   - judge output malformed (armJudge never re-asks) -> judge_error
  *     cell: excluded from aggregates, surfaced in the run report, never
  *     silently dropped.
  *
@@ -108,7 +108,7 @@ async function runArmCell(task: ArmTask, arm: Arm): Promise<CellResult> {
     }
     const checks = runChecks(task, dirs.dir);
 
-    // Judge taxonomy: still malformed after armJudge's bounded retries ->
+    // Judge taxonomy: malformed armJudge output (no re-ask) ->
     // judge_error cell (excluded from aggregates, surfaced in the report).
     let judge: ArmJudgeScore | null = null;
     let judgeError: string | null = null;

test/skill-e2e-qa-callers.test.ts

@@ -79,10 +79,17 @@ async function capture(caseId: QaCallerCase) {
   }
 }
 
-describeIfSelected('Automatic parent exploratory QA', [...QA_CALLER_CASES], () => {
-  for (const caseId of QA_CALLER_CASES) {
-    testConcurrentIfSelected(caseId, () => capture(caseId), QA_CALLER_TEST_MS);
-  }
+const LITERAL_CASES = ['review-exploratory-small-cli', 'ship-exploratory-small-cli', 'ship-exploratory-unavailable',
+  'ship-exploratory-plan-checks', 'ship-exploratory-late-input'] as const;
+if (LITERAL_CASES.join() !== QA_CALLER_CASES.join()) throw new Error('qa-callers literal registration drifted from QA_CALLER_CASES');
+
+describeIfSelected('Automatic parent exploratory QA', ['review-exploratory-small-cli', 'ship-exploratory-small-cli',
+  'ship-exploratory-unavailable', 'ship-exploratory-plan-checks', 'ship-exploratory-late-input'], () => {
+  testConcurrentIfSelected('review-exploratory-small-cli', () => capture('review-exploratory-small-cli'), QA_CALLER_TEST_MS);
+  testConcurrentIfSelected('ship-exploratory-small-cli', () => capture('ship-exploratory-small-cli'), QA_CALLER_TEST_MS);
+  testConcurrentIfSelected('ship-exploratory-unavailable', () => capture('ship-exploratory-unavailable'), QA_CALLER_TEST_MS);
+  testConcurrentIfSelected('ship-exploratory-plan-checks', () => capture('ship-exploratory-plan-checks'), QA_CALLER_TEST_MS);
+  testConcurrentIfSelected('ship-exploratory-late-input', () => capture('ship-exploratory-late-input'), QA_CALLER_TEST_MS);
 });
 
 afterAll(() => finalizeEvalCollector(collector));